Application of deep supervised learning methods to identify the position of active/inactive cis-regulatory regions.
Note: The following code takes inspiration from the lab notebooks but was very much deconstructed, revised and customized for a better personal understanding, at times, even at the cost of some compactness and encapsulation present in the original.
from epigenomic_dataset import load_epigenomes
import numpy as np
import pandas as pd
from sklearn.impute import KNNImputer
from sklearn.preprocessing import RobustScaler
from scipy.stats import pearsonr
from scipy.stats import spearmanr
from minepy import MINE
from scipy.stats import entropy
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from boruta import BorutaPy
from multiprocessing import cpu_count
from sklearn.decomposition import PCA as skPCA
from sklearn.manifold import TSNE as skTSNE
import matplotlib.pyplot as plt
from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
import tensorflow as tf
from tensorflow.keras import activations, layers, optimizers, losses
from tensorflow.keras import Sequential
from extra_keras_metrics import get_standard_binary_metrics
from tensorflow.keras.callbacks import EarlyStopping
from sklearn.metrics import accuracy_score, average_precision_score, roc_auc_score
#Genomic
from ucsc_genomes_downloader import Genome
from keras_bed_sequence import BedSequence
from keras_mixed_sequence import MixedSequence, VectorSequence
import keras_tuner as kt
Load and visualize epigenomic data (features+labels) for both promoters and enhancers for cell line HepG2.
It is necessary to convert the real valued labels into discrete binary values to treat the problem as a binary classification task. Hence a threshold is selected to label regions as active or inactive.
Since this binarization is still an open problem and the literature doesn't provide a clear method to perform it, nor a specific cut-off value, the choice of the threshold was guided by the necessity to reach a reasonable balance between the classes and the will to not discard any piece of data.
cell_line = "HEPG2"
window_size = 256
promoters_epigenomes, promoters_labels = load_epigenomes(
cell_line = cell_line,
dataset = "fantom",
region = "promoters",
window_size = window_size,
binarize = True,
min_active_tpm_value = 1,
max_inactive_tpm_value = 1
)
enhancers_epigenomes, enhancers_labels = load_epigenomes(
cell_line = cell_line,
dataset = "fantom",
region = "enhancers",
window_size = window_size,
binarize = True,
min_active_tpm_value = 0.09,
max_inactive_tpm_value = 0.09
)
print(f"Promoters' features shape: {promoters_epigenomes.shape}")
print(f"Promoters' labels shape: {promoters_labels.shape}")
print(f"Enhancers' features shape: {enhancers_epigenomes.shape}")
print(f"Enhancers' labels shape: {enhancers_labels.shape}")
Promoters' features shape: (99881, 562) Promoters' labels shape: (99881, 1) Enhancers' features shape: (63285, 562) Enhancers' labels shape: (63285, 1)
print(f"Inactive promoters: {promoters_labels[promoters_labels['HEPG2'] == False].size}\nActive promoters: {promoters_labels[promoters_labels['HEPG2'] == True].size}")
print(f"Inactive enhancers: {enhancers_labels[enhancers_labels['HEPG2'] == False].size}\nActive enhancers: {enhancers_labels[enhancers_labels['HEPG2'] == True].size}")
Inactive promoters: 74099 Active promoters: 25782 Inactive enhancers: 56108 Active enhancers: 7177
promoters_epigenomes[:5]
| chrom | NR2F6 | ZCCHC11 | PPARG | AGO2 | CEBPZ | SP2 | ZNF510 | CREB1 | ZNF526 | ZNF629 | ... | ZNF224 | E2F4 | NR3C1 | SAFB2 | CHD4 | ZNF232 | ZNF788 | NCOR1 | PREB | ZNF576 | |||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| chrom | chromStart | chromEnd | strand | |||||||||||||||||||||
| chr1 | 628964 | 629220 | + | 0.00 | 0.90 | 0.34 | 0.67 | 0.0 | 0.64 | 0.82 | 0.40 | 0.27 | 0.56 | ... | 0.09 | 0.34 | 0.24 | 1.06 | 0.14 | 0.31 | 0.00 | 0.34 | 0.29 | 0.69 |
| 629013 | 629269 | + | 0.06 | 1.22 | 0.51 | 0.67 | 0.0 | 0.66 | 0.81 | 0.42 | 0.27 | 0.43 | ... | 0.17 | 0.44 | 0.18 | 1.17 | 0.08 | 0.41 | 0.00 | 0.33 | 0.33 | 0.58 | |
| 629642 | 629898 | + | 0.00 | 0.41 | 0.57 | 0.19 | 0.0 | 0.00 | 0.16 | 0.03 | 0.17 | 0.15 | ... | 0.00 | 0.05 | 0.00 | 0.00 | 0.01 | 0.24 | 0.00 | 0.34 | 0.21 | 0.06 | |
| 629847 | 630103 | + | 0.68 | 1.75 | 1.11 | 0.13 | 0.0 | 0.35 | 1.31 | 0.91 | 0.75 | 0.38 | ... | 0.05 | 0.31 | 0.70 | 0.59 | 0.00 | 0.42 | 0.24 | 0.54 | 0.25 | 0.51 | |
| 629905 | 630161 | + | 0.68 | 1.81 | 0.98 | 0.04 | 0.0 | 0.59 | 1.25 | 1.21 | 0.80 | 0.47 | ... | 0.14 | 0.36 | 1.18 | 0.62 | 0.00 | 0.49 | 0.28 | 0.39 | 0.31 | 0.57 |
5 rows × 562 columns
promoters_epigenomes.describe()
| chrom | NR2F6 | ZCCHC11 | PPARG | AGO2 | CEBPZ | SP2 | ZNF510 | CREB1 | ZNF526 | ZNF629 | ... | ZNF224 | E2F4 | NR3C1 | SAFB2 | CHD4 | ZNF232 | ZNF788 | NCOR1 | PREB | ZNF576 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99880.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | ... | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 | 99881.000000 |
| mean | 1.097180 | 1.918595 | 2.028786 | 5.230861 | 0.714952 | 1.978142 | 1.398598 | 2.460228 | 1.653768 | 1.335839 | ... | 1.186699 | 3.205277 | 1.051440 | 1.837787 | 1.502751 | 1.541730 | 1.947397 | 1.641510 | 1.104601 | 1.146580 |
| std | 1.424951 | 1.540896 | 2.248470 | 8.876331 | 1.288516 | 3.746221 | 0.991979 | 9.593309 | 3.292122 | 1.376866 | ... | 0.685587 | 5.167217 | 0.489308 | 1.425254 | 1.194455 | 1.166041 | 1.568245 | 1.742847 | 0.604027 | 0.618647 |
| min | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ... | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
| 25% | 0.450000 | 0.730000 | 0.590000 | 0.620000 | 0.320000 | 0.660000 | 0.670000 | 0.520000 | 0.690000 | 0.630000 | ... | 0.690000 | 0.630000 | 0.710000 | 0.850000 | 0.640000 | 0.710000 | 0.800000 | 0.590000 | 0.690000 | 0.730000 |
| 50% | 0.790000 | 1.390000 | 1.240000 | 1.980000 | 0.580000 | 1.040000 | 1.150000 | 0.800000 | 1.150000 | 1.040000 | ... | 1.100000 | 1.250000 | 0.990000 | 1.460000 | 1.200000 | 1.220000 | 1.500000 | 1.070000 | 1.020000 | 1.080000 |
| 75% | 1.280000 | 2.900000 | 2.920000 | 6.050000 | 0.890000 | 1.660000 | 1.940000 | 1.500000 | 1.810000 | 1.700000 | ... | 1.570000 | 3.770000 | 1.330000 | 2.440000 | 2.070000 | 2.100000 | 2.740000 | 2.160000 | 1.420000 | 1.480000 |
| max | 46.970000 | 24.070000 | 73.100000 | 172.780000 | 73.090000 | 68.750000 | 15.150000 | 677.610000 | 120.840000 | 50.290000 | ... | 7.750000 | 98.620000 | 7.400000 | 25.110000 | 21.100000 | 22.860000 | 17.940000 | 30.980000 | 8.820000 | 23.520000 |
8 rows × 562 columns
promoters_labels[:5]
| HEPG2 | ||||
|---|---|---|---|---|
| chrom | chromStart | chromEnd | strand | |
| chr1 | 628964 | 629220 | + | False |
| 629013 | 629269 | + | False | |
| 629642 | 629898 | + | False | |
| 629847 | 630103 | + | False | |
| 629905 | 630161 | + | True |
enhancers_epigenomes[:5]
| chrom | NR2F6 | ZCCHC11 | PPARG | AGO2 | CEBPZ | SP2 | ZNF510 | CREB1 | ZNF526 | ZNF629 | ... | ZNF224 | E2F4 | NR3C1 | SAFB2 | CHD4 | ZNF232 | ZNF788 | NCOR1 | PREB | ZNF576 | |||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| chrom | chromStart | chromEnd | strand | |||||||||||||||||||||
| chr10 | 100006381 | 100006637 | . | 1.35 | 1.62 | 1.43 | 0.36 | 1.24 | 0.38 | 0.72 | 2.21 | 1.01 | 0.68 | ... | 0.78 | 1.12 | 0.47 | 1.13 | 1.35 | 0.93 | 0.94 | 1.96 | 0.87 | 0.79 |
| 100008146 | 100008402 | . | 0.60 | 0.81 | 0.92 | 0.44 | 0.24 | 0.44 | 1.02 | 0.36 | 0.27 | 0.91 | ... | 0.64 | 0.62 | 0.14 | 0.88 | 1.19 | 0.87 | 0.63 | 1.28 | 0.76 | 0.76 | |
| 100014418 | 100014674 | . | 1.60 | 1.41 | 1.48 | 0.18 | 0.59 | 1.32 | 0.99 | 0.94 | 1.29 | 0.85 | ... | 2.52 | 1.15 | 0.90 | 1.17 | 2.43 | 1.29 | 1.47 | 0.56 | 0.82 | 1.28 | |
| 100020216 | 100020472 | . | 1.99 | 2.08 | 5.10 | 0.15 | 0.76 | 0.71 | 2.17 | 1.78 | 1.18 | 1.12 | ... | 0.94 | 1.29 | 0.56 | 1.21 | 2.22 | 1.66 | 1.79 | 1.93 | 0.23 | 1.14 | |
| 100043528 | 100043784 | . | 0.97 | 0.92 | 1.86 | 1.42 | 0.63 | 0.94 | 1.05 | 0.87 | 1.10 | 0.70 | ... | 1.23 | 0.83 | 0.86 | 0.34 | 1.00 | 0.78 | 0.61 | 0.26 | 1.54 | 0.69 |
5 rows × 562 columns
enhancers_epigenomes.describe()
| chrom | NR2F6 | ZCCHC11 | PPARG | AGO2 | CEBPZ | SP2 | ZNF510 | CREB1 | ZNF526 | ZNF629 | ... | ZNF224 | E2F4 | NR3C1 | SAFB2 | CHD4 | ZNF232 | ZNF788 | NCOR1 | PREB | ZNF576 |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | ... | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 | 63285.000000 |
| mean | 0.963189 | 1.059173 | 1.088036 | 1.208639 | 0.771997 | 0.800150 | 0.876783 | 0.939837 | 0.877640 | 0.875734 | ... | 0.851586 | 0.867810 | 0.861504 | 0.945402 | 1.079634 | 0.903154 | 0.976611 | 1.054499 | 0.835609 | 0.878923 |
| std | 1.397075 | 0.803125 | 1.638099 | 3.221310 | 0.453335 | 0.818764 | 0.570115 | 2.168628 | 0.674322 | 0.974294 | ... | 0.530374 | 1.004479 | 0.375902 | 0.655873 | 1.016449 | 0.656338 | 0.833855 | 1.415519 | 0.462954 | 0.521006 |
| min | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | ... | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
| 25% | 0.450000 | 0.550000 | 0.450000 | 0.230000 | 0.470000 | 0.470000 | 0.500000 | 0.520000 | 0.490000 | 0.470000 | ... | 0.470000 | 0.460000 | 0.600000 | 0.490000 | 0.500000 | 0.500000 | 0.430000 | 0.510000 | 0.510000 | 0.510000 |
| 50% | 0.700000 | 0.860000 | 0.720000 | 0.500000 | 0.730000 | 0.720000 | 0.770000 | 0.720000 | 0.790000 | 0.710000 | ... | 0.780000 | 0.690000 | 0.830000 | 0.830000 | 0.790000 | 0.790000 | 0.800000 | 0.780000 | 0.790000 | 0.830000 |
| 75% | 1.040000 | 1.300000 | 1.140000 | 1.020000 | 1.020000 | 1.010000 | 1.120000 | 0.970000 | 1.140000 | 1.030000 | ... | 1.140000 | 1.000000 | 1.090000 | 1.250000 | 1.270000 | 1.150000 | 1.290000 | 1.140000 | 1.100000 | 1.180000 |
| max | 60.590000 | 17.830000 | 70.630000 | 138.330000 | 27.970000 | 50.300000 | 8.480000 | 195.180000 | 43.040000 | 43.740000 | ... | 14.140000 | 55.770000 | 4.620000 | 9.340000 | 18.070000 | 35.260000 | 11.190000 | 40.940000 | 5.400000 | 28.750000 |
8 rows × 562 columns
enhancers_labels[:5]
| HEPG2 | ||||
|---|---|---|---|---|
| chrom | chromStart | chromEnd | strand | |
| chr10 | 100006381 | 100006637 | . | True |
| 100008146 | 100008402 | . | False | |
| 100014418 | 100014674 | . | False | |
| 100020216 | 100020472 | . | False | |
| 100043528 | 100043784 | . | False |
feature_number = promoters_epigenomes.shape[1]
If the number of features is higher than the number of samples the model will behave erratically and overfit.
print(f"Promoters samples-features ratio: {promoters_epigenomes.shape[0]/promoters_epigenomes.shape[1]}")
print(f"Enhancers samples-features ratio: {enhancers_epigenomes.shape[0]/enhancers_epigenomes.shape[1]}")
Promoters samples-features ratio: 177.72419928825622 Enhancers samples-features ratio: 112.6067615658363
NaN values have a detrimental impact on the performance of the model.
print("NaN values report for Promoters: ")
print(f"Total NaN values: {promoters_epigenomes.isna().values.sum()} out of {promoters_epigenomes.values.size} values")
print(f"Row (sample) with the most NaN values has {promoters_epigenomes.isna().sum(axis=0).max()} NaN values out of {promoters_epigenomes.shape[1]} NaN values")
print(f"Column (feature) with the most NaN values has {promoters_epigenomes.isna().sum().max()} NaN values out of {promoters_epigenomes.shape[0]} NaN values")
print("-"*85)
print("NaN values report for Enhancers: ")
print(f"Total NaN values: {enhancers_epigenomes.isna().values.sum()} out of {enhancers_epigenomes.values.size} values")
print(f"Row (sample) with the most NaN values has {enhancers_epigenomes.isna().sum(axis=0).max()} NaN values out of {enhancers_epigenomes.shape[1]} NaN values")
print(f"Column (feature) with the most NaN values has {enhancers_epigenomes.isna().sum().max()} NaN values out of {enhancers_epigenomes.shape[0]} NaN values")
NaN values report for Promoters: Total NaN values: 359 out of 56133122 values Row (sample) with the most NaN values has 138 NaN values out of 562 NaN values Column (feature) with the most NaN values has 138 NaN values out of 99881 NaN values ------------------------------------------------------------------------------------- NaN values report for Enhancers: Total NaN values: 19 out of 35566170 values Row (sample) with the most NaN values has 19 NaN values out of 562 NaN values Column (feature) with the most NaN values has 19 NaN values out of 63285 NaN values
The amount of NaN values is small enough to allow imputation.
def knn_imputation(df: pd.DataFrame, n) -> pd.DataFrame:
""" Return the provided dataframe with NaN values imputed using knn inputation.
Parameters
------------------------
df: pd.DataFrame
The dataframe to impute.
n: int
The number of neighbours to consider.
Returns
------------------------
pd.DataFrame
The dataframe with the NaN values imputed."""
return pd.DataFrame(
KNNImputer(n_neighbors=n).fit_transform(df.values),
columns=df.columns,
index=df.index
)
n = 5
promoters_epigenomes = knn_imputation(promoters_epigenomes, n)
enhancers_epigenomes = knn_imputation(enhancers_epigenomes, n)
print(f"Promoters NaN values: {promoters_epigenomes.isna().values.sum()}")
print(f"Enhancers NaN values: {enhancers_epigenomes.isna().values.sum()}")
Promoters NaN values: 0 Enhancers NaN values: 0
Visualizing the data helps to evaluate the difficulty of the task and to design models with the appropiate complexity. In this case data decomposition techniques (PCA and t-SNE) are used to reduce the high dimensional dataset into fewer dimensions (two), while preserving spatial characteristics for visualization purposes.
In this portion of the code epigenomic data is being pre-processed, however, for this visualization part, sequence data is also considered.
Technique to reduce the dimensionality of a dataset by linearly transforming the data into a new coordinate system where the variation in the data can be described with fewer dimensions than the initial data while preserving most of the original data variation.
Non linear dimensionality reduction technique, based on Stochastic Neighbour Embedding, well-suited for embedding high-dimensional data in a low-dimensional space of two dimensions for visualization.
# Importing the genome, for the visualization of sequence data
genome = Genome("hg38")
def to_bed(data:pd.DataFrame)->pd.DataFrame:
"""Return bed coordinates from given dataset."""
return data.reset_index()[data.index.names]
from sklearn.decomposition import PCA
def get_pca_decomposition(
X: pd.DataFrame,
data_type: str,
region: str,
cell_line: str,
metric: str,
**kwargs
) -> pd.DataFrame:
"""Return the 2D TSNE decomposition of the given data."""
return pd.DataFrame(
PCA(n_components=2).fit_transform(X.values),
index=X.index
)
def get_tsne_decomposition(
X: pd.DataFrame,
data_type: str,
region: str,
cell_line: str,
metric: str,
)->pd.DataFrame:
"""Return the 2D TSNE decomposition of the given data."""
from MulticoreTSNE import MulticoreTSNE as TSNE
return pd.DataFrame(
TSNE(verbose=True).fit_transform(X.values),
index=X.index
)
from matplotlib.figure import Figure
from matplotlib.axes import Axes
from matplotlib.colors import ListedColormap, LogNorm
def scatter_with_heatmap(
decomposition: pd.DataFrame,
scores: np.ndarray,
figure: Figure,
ax: Axes
):
"""Plot the scatter plot."""
scatter = ax.scatter(
*decomposition.values.T,
c=scores,
norm=matplotlib.colors.Normalize(vmin=-1, vmax=1),
s=3
)
color_bar = figure.colorbar(scatter, ax=ax)
The following loop plots PCA and t-SNE decomposition for epigenomic data and sequence data considering both promoters and enhancers.
Note: since the labels are binarized the max/mean metric differentiation is useless and produces the same result twice. Due to the computational time required to run t-SNE however, this cell has not been touched in that regard.
epigenomic_fig, epigenomic_axes = plt.subplots(
nrows=8,
ncols=1,
figsize=(10, 80),
squeeze=True
)
sequence_fig, sequence_axes = plt.subplots(
nrows=4,
ncols=1,
figsize=(10, 40),
squeeze=True
)
for k, metric in enumerate(("max", "mean")):
for l, decomposition_method in enumerate((get_tsne_decomposition, get_pca_decomposition)):
for j, ((epigenomic, scores), region, clip_value) in enumerate((
((promoters_epigenomes, promoters_labels), "Promoters", 1000),
((enhancers_epigenomes, enhancers_labels), "Enhancers", 20)
)):
# Epigenomic data
decomposed_epigenomic = decomposition_method(
epigenomic,
"epigenomic",
region,
cell_line,
metric
)
scores[scores>clip_value] = clip_value
scatter_with_heatmap(
decomposition=decomposed_epigenomic,
scores=scores.values,
figure=epigenomic_fig,
ax=epigenomic_axes[j + 2*k + l*4]
)
epigenomic_axes[j + 2*k + l*4].set_title(f"{cell_line} - {region} {metric} HG38, epigenomic data")
epigenomic_axes[j + 2*k + l*4].set_axis_off()
# Sequence data
if k==0:
sequence = pd.DataFrame(
np.array(BedSequence(
genome,
bed=to_bed(scores),
batch_size=1
)).reshape(-1, 4*256),
index=scores.index
)
decomposed_sequence = decomposition_method(
sequence,
"sequence",
region,
cell_line,
metric="None",
)
scatter_with_heatmap(
decomposition=decomposed_sequence,
scores=scores.values,
figure=sequence_fig,
ax=sequence_axes[j + l*2]
)
sequence_axes[j + l*2].set_title(f"{cell_line} - {region} HG38, sequence data")
sequence_axes[j + l*2].set_axis_off()
epigenomic_fig.tight_layout()
sequence_fig.tight_layout()
plt.show()
Data normalization is proven to improve the performance of models.
The robust scaler operates normalization subtracting the median and dividing by the standard deviation between the interquantile range. This is operates a z-scoring while being robust to outliers.
def robust_scaler(df: pd.DataFrame) -> pd.DataFrame:
""" Return the provided dataframe normalized using robust z-scoring.
Parameters
------------------------
df: pd.DataFrame
The dataframe to normalize.
Returns
------------------------
pd.DataFrame
The dataframe normalized."""
return pd.DataFrame(
RobustScaler().fit_transform(df.values),
columns=df.columns,
index=df.index
)
promoters_epigenomes = robust_scaler(promoters_epigenomes)
enhancers_epigenomes = robust_scaler(enhancers_epigenomes)
Feature selection implies the choice of a subset of the original pool of features.
This is a useful step as irrelevant or partially relevant features can negatively impact the model performance, hence feature selection reduces noise, improves model accuracy and reduces training time by cutting the total number of features (currently pretty high).
Features not correlated with the output do not carry relevan information for the model to exploit for the task at hand. These can be removed.
In the following section functions are defined and ran for each of these tests over the entire dataset. This is done here just for explanatory purposes, in fact the resulting correlated features are not dropped from the dataset as this would introduce bias due to data leakage. Instead these tests are saved for the training "main loop" and ran, for each holdout, over the training portion of the data only, maintaining test data unseen and reserving it for actual testing.
p_value_threshold = 0.01
correlation_threshold = 0.05
def get_pearson_uncorrelated(
data: pd.DataFrame,
labels: pd.DataFrame,
p_value_threshold: float = 0.01
) -> list:
""" Return the list of features that the Pearson correlation test found to be uncorrelated with the labels.
Parameters
------------------------
data: pd.DataFrame
The dataset (epigenomic data).
labels: pd.dataFrame
The data labels.
p_value_threshold: float = 0.01
The minimum p-value required to consider a feature uncorrelated.
Returns
------------------------
list: list
The list of uncorrelated features."""
uncorrelated_features = []
for feature in (data.columns):
correlation, p_value = pearsonr(data[feature].values.ravel(), labels.values.ravel())
if p_value > p_value_threshold:
if feature not in uncorrelated_features:
uncorrelated_features.append(feature)
return uncorrelated_features
print("-- Just for explanatory purpose --")
uncorrelated_features_pearson_promoters = get_pearson_uncorrelated(promoters_epigenomes, promoters_labels)
uncorrelated_features_pearson_enhancers = get_pearson_uncorrelated(enhancers_epigenomes, enhancers_labels)
print(f"Promoters: features not correlated with the output: [Pearson]\n{uncorrelated_features_pearson_promoters}")
print("-"*80)
print(f"Enhancers: features not correlated with the output: [Pearson]\n{uncorrelated_features_pearson_enhancers}")
-- Just for explanatory purpose -- Promoters: features not correlated with the output: [Pearson] [] -------------------------------------------------------------------------------- Enhancers: features not correlated with the output: [Pearson] ['SNRNP70']
def get_spearman_uncorrelated(
data: pd.DataFrame,
labels: pd.DataFrame,
p_value_threshold: float = 0.01
) -> list:
""" Return the list of features that the Spearman correlation test found to be uncorrelated with the labels.
Parameters
------------------------
data: pd.DataFrame
The dataset (epigenomic data).
labels: pd.dataFrame
The data labels.
p_value_threshold: float = 0.01
The minimum p-value required to consider a feature uncorrelated.
Returns
------------------------
list: list
The list of uncorrelated features."""
uncorrelated_features = []
for feature in (data.columns):
correlation, p_value = spearmanr(data[feature].values.ravel(), labels.values.ravel())
if p_value > p_value_threshold:
if feature not in uncorrelated_features:
uncorrelated_features.append(feature)
return uncorrelated_features
print("-- Just for explanatory purpose --")
uncorrelated_features_spearman_promoters = get_spearman_uncorrelated(promoters_epigenomes, promoters_labels)
uncorrelated_features_spearman_enhancers = get_spearman_uncorrelated(enhancers_epigenomes, enhancers_labels)
print(f"Promoters: features not correlated with the output: [Spearman]\n{uncorrelated_features_spearman_promoters}")
print("-"*80)
print(f"Enhancers: features not correlated with the output: [Spearman]\n{uncorrelated_features_spearman_enhancers}")
-- Just for explanatory purpose -- Promoters: features not correlated with the output: [Spearman] ['ZNF207', 'ZNF737'] -------------------------------------------------------------------------------- Enhancers: features not correlated with the output: [Spearman] ['SNRNP70']
print("-- Just for explanatory purpose --")
uncorrelated_features_promoters = uncorrelated_features_pearson_promoters + uncorrelated_features_spearman_promoters
uncorrelated_features_enhancers = uncorrelated_features_pearson_enhancers + uncorrelated_features_spearman_enhancers
uncorrelated_features_promoters = list(set(uncorrelated_features_promoters))
uncorrelated_features_enhancers = list(set(uncorrelated_features_enhancers))
print(f"Promoters: currently there are {len(uncorrelated_features_promoters)} uncorrelated features with the output:\n{uncorrelated_features_promoters}" )
print("-"*80)
print(f"Enhancers: currently there are {len(uncorrelated_features_enhancers)} uncorrelated features with the output:\n{uncorrelated_features_enhancers}" )
-- Just for explanatory purpose -- Promoters: currently there are 2 uncorrelated features with the output: ['ZNF737', 'ZNF207'] -------------------------------------------------------------------------------- Enhancers: currently there are 1 uncorrelated features with the output: ['SNRNP70']
def get_mic_uncorrelated(
data: pd.DataFrame,
labels: pd.DataFrame,
features_to_check: list,
correlation_threshold: float = 0.05
) -> list:
""" Return the list of features that the MIC correlation test confirmed to be uncorrelated with the labels.
Parameters
------------------------
data: pd.DataFrame
The dataset (epigenomic data).
labels: pd.dataFrame
The data labels.
features_to_check: list
The list of features to check with the MIC correlation test.
correlation_threshold: float = 0.05
The maximum correlation value required to consider a feature uncorrelated.
Returns
------------------------
list: list
The list of uncorrelated features."""
uncorrelated_features = []
for feature in (features_to_check):
mine = MINE()
mine.compute_score(data[feature].values.ravel(), labels.values.ravel())
score = mine.mic()
if score < correlation_threshold:
uncorrelated_features.append(feature)
return uncorrelated_features
print("-- Just for explanatory purpose --")
uncorrelated_features_mic_promoters = get_mic_uncorrelated(promoters_epigenomes, promoters_labels, uncorrelated_features_promoters)
uncorrelated_features_mic_enhancers = get_mic_uncorrelated(enhancers_epigenomes, enhancers_labels, uncorrelated_features_enhancers)
print(f"Promoters: there are {len(uncorrelated_features_mic_promoters)} uncorrelated features with the output: [Confirmed via MIC]\n{uncorrelated_features_mic_promoters}" )
print("-"*80)
print(f"Enhancers: there are {len(uncorrelated_features_mic_enhancers)} uncorrelated features with the output: [Confirmed via MIC]\n{uncorrelated_features_mic_enhancers}" )
-- Just for explanatory purpose -- Promoters: there are 2 uncorrelated features with the output: [Confirmed via MIC] ['ZNF737', 'ZNF207'] -------------------------------------------------------------------------------- Enhancers: there are 1 uncorrelated features with the output: [Confirmed via MIC] ['SNRNP70']
def uncorrelated_features_test(
data: pd.DataFrame,
labels: pd.DataFrame,
p_value_threshold: float = 0.01,
correlation_threshold: float = 0.05
) -> list:
""" Return the list of features that the Pearson+Spearman+Mic correlation tests
found to be uncorrelated with the output.
Parameters
------------------------
data: pd.DataFrame
The dataset (epigenomic data).
labels: pd.dataFrame
The data labels.
p_value_threshold: float = 0.01
The minimum p-value required to consider a feature uncorrelated.
correlation_threshold: float = 0.05
The maximum correlation value required to consider a feature uncorrelated.
Returns
------------------------
list: list
The list of uncorrelated features."""
uncorrelated_features = []
#Pearson
uncorrelated_features += get_pearson_uncorrelated(data, labels, p_value_threshold)
#Spearman
uncorrelated_features += get_spearman_uncorrelated(data, labels, p_value_threshold)
uncorrelated_features = list(set(uncorrelated_features))
print(f"-- {len(uncorrelated_features)} Uncorrelated features: [Pearson+Spearman]\n{uncorrelated_features}")
actually_uncorrelated_features = uncorrelated_features
#Mic
actually_uncorrelated_features = get_mic_uncorrelated(data, labels, uncorrelated_features, correlation_threshold)
print(f"-- Actually uncorrelated features: [confirmed via MIC]\n{actually_uncorrelated_features}")
return actually_uncorrelated_features
Features correlated with each other carry the same kind of information. These can be removed.
Identifies linear correlation between features.
def correlated_features_test(
data: pd.DataFrame,
p_value_threshold: float = 0.01,
correlation_threshold: float = 0.97
) -> list:
""" Return the features that the Pearson correlation tests found to be correlated with other features.
Parameters
------------------------
data: pd.DataFrame
The dataset (epigenomic data).
p_value_threshold: float = 0.01
The maximum p-value required to consider a feature correlated.
correlation_threshold: float = 0.97
The minimum correlation value required to consider a feature correlated.
Returns
------------------------
list: list
The list of correlated features."""
correlated_features = []
#Pearson
for i, feature1 in (enumerate(data.columns)):
for feature2 in data.columns[i+1:]:
correlation, p_value = pearsonr(data[feature1].values.ravel(), data[feature2].values.ravel())
correlation = np.abs(correlation)
if p_value < p_value_threshold and correlation > correlation_threshold:
if entropy(data[feature1]) > entropy(data[feature2]):
correlated_features.append(feature2)
else:
correlated_features.append(feature1)
correlated_features = list(set(correlated_features))
print(f"-- {len(correlated_features)} Correlated features: [Pearson]\n{correlated_features}")
return correlated_features
print("-- Just for explanatory purpose --")
correlated_features_promoters = correlated_features_test(promoters_epigenomes)
correlated_features_enhancers = correlated_features_test(enhancers_epigenomes)
-- Just for explanatory purpose -- -- 1 Correlated features: [Pearson] ['SAP130'] -- 0 Correlated features: [Pearson] []
The algorithm gives a numerical estimate of the feature importance and categorizes features in those to keep, those to discard and tentative, meaning that there is uncertainty regarding their actual correlation with the output. Both uncorrelated and tentative features are discarded. Experiments were performed when discarding only strictly uncorrelated features, the results, however, were found to be similar.
def boruta_feature_selection(
data: pd.DataFrame,
labels: pd.DataFrame,
max_iter: int = 100
) -> pd.DataFrame:
""" Return the features that the Boruta algorithm found to be correlated with the output (useful/to keep).
Parameters
------------------------
data: pd.DataFrame
The data reserved for the input of the training of the Boruta model.
labels: pd.DataFrame
The data reserved for the output of the training of the Boruta model.
max_iter: int = 100
The number of iterations to run Boruta for.
Returns
------------------------
list: list
The list of correlated features."""
# Create the Boruta model
boruta_selector = BorutaPy(
RandomForestClassifier(n_jobs=cpu_count(), class_weight='balanced', max_depth=5),
n_estimators = 'auto',
verbose = 2,
alpha = 0.05,
max_iter = max_iter,
random_state = 42
)
# Fit the Boruta model
boruta_selector.fit(data.values, labels.values.ravel())
# Get the kept features and discarded features
kept_features = list(data.columns[boruta_selector.support_])
discarted_features = list(data.columns[~boruta_selector.support_])
print(f"-- {len(kept_features)} Kept (correlated) features [Boruta]")
print(f"-- {len(discarted_features)} Discarted (uncorrelated) features [Boruta]")
return kept_features
The technique used to generate the holdouts for the evaluation of the various architectures is the stratified Monte-Carlo method, producing each time a different arrangement of the training, validation and testing set, while keeping roughly the same class balance.
10 holdouts are considered to train and evaluate each model, 20% of the whole dataset was used for testing, 80% for training where 20% of it is reserved for the validation of the meta-model.
To avoid increasing excessively the computational complexity of the main training loop, the hyperparameter tuning step is performed, for all holdouts, considering a single split of the training set in a training and validation portion.
number_of_splits = 10
holdouts_generator = StratifiedShuffleSplit(
n_splits = number_of_splits,
test_size = 0.2,
random_state = 42
)
number_of_splits_tuning = 1
holdouts_generator_tuning = StratifiedShuffleSplit(
n_splits = number_of_splits_tuning,
test_size = 0.2,
random_state = 42
)
def train_model(
model: tf.keras.Model,
x_train: np.ndarray,
x_test: np.ndarray,
y_train: np.ndarray,
y_test: np.ndarray,
epochs: int,
batch_size: int
):
""" Train the model and return the performance metrics (accuracy, AUROC, AUPRC) for train and test and the training history.
Parameters
------------------------
x_train: np.ndarray
The input data for training.
x_test: np.ndarray
The input data for testing.
y_train: np.ndarray
The labels of the input data for training.
y_test: np.ndarray
The labels of the input data for testing.
epochs: int
The number of times the learning algorithm works through the dataset.
batch_size: int
The number of samples to work through before updating the model.
Returns
------------------------
The performance metrics and history of the model."""
model_history = model.fit(
x_train, y_train,
validation_data = (x_test, y_test),
epochs = epochs,
batch_size = batch_size,
callbacks = [EarlyStopping("val_loss", patience=2)]
)
y_train_pred = model.predict(x_train)
y_test_pred = model.predict(x_test)
metrics = evaluate_model_prediction(y_train, y_train_pred, y_test, y_test_pred)
return metrics, model_history
Next are a set of functions to evaluate and visualize the performances of models.
To have a statistically sound estimate of an architecture performance, multiple models are built and trained, each with the same architecture, over different portions of the data (holdouts) and, the average performance of those, is considered as an estimate of the overall performance of the architecture.
def evaluate_model_prediction(
y_train_true: np.ndarray,
y_train_pred: np.ndarray,
y_test_true: np.ndarray,
y_test_pred: np.ndarray
) -> dict[str, float]:
""" Return the performance metrics (accuracy, AUROC, AUPRC) for train and test of the model predictions.
Parameters
------------------------
y_train_true: np.ndarray
The actual labels (train).
y_train_pred: np.ndarray
The predicted labels (train).
y_test_true: np.ndarray
The actual labels (test).
y_test_pred: np.ndarray
The predicted labels (test).
Returns
------------------------
dict: dict[str, float]
The performance metrics of the model predictions."""
accuracy_train = accuracy_score(y_train_true, np.round(y_train_pred).astype(int))
AUROC_train = roc_auc_score(y_train_true, y_train_pred)
AUPRC_train = average_precision_score(y_train_true, y_train_pred)
accuracy_test = accuracy_score(y_test_true, np.round(y_test_pred).astype(int))
AUROC_test = roc_auc_score(y_test_true, y_test_pred)
AUPRC_test = average_precision_score(y_test_true, y_test_pred)
metrics = {
"accuracy_train": accuracy_train,
"accuracy_test": accuracy_test,
"AUROC_train": AUROC_train,
"AUROC_test": AUROC_test,
"AUPRC_train": AUPRC_train,
"AUPRC_test": AUPRC_test
}
return metrics
def model_metrics_holdout_estimate(
model_metrics: list,
holdout_total_number: int
) -> dict[str, float]:
""" Return the average metrics (accuracy, AUROC, AUPRC) of the model accross all the holdouts for train and test.
Parameters
------------------------
model_metrics: list
The list with the train/test metrics (accuracy, AUROC, AUPRC) for all the holdouts.
holdout_total_number: int
The number of total holdouts.
Returns
------------------------
dict: dict[str, float]
The average metrics of the model accross all the holdouts for train and test."""
total_accuracy_train = 0
total_accuracy_test = 0
total_AUROC_train = 0
total_AUROC_test = 0
total_AUPRC_train = 0
total_AUPRC_test = 0
for holdout in model_metrics:
total_accuracy_train += holdout['accuracy_train']
total_accuracy_test += holdout['accuracy_test']
total_AUROC_train += holdout['AUROC_train']
total_AUROC_test += holdout['AUROC_test']
total_AUPRC_train += holdout['AUPRC_train']
total_AUPRC_test += holdout['AUPRC_test']
avg_accuracy_train = total_accuracy_train / holdout_total_number
avg_accuracy_test = total_accuracy_test / holdout_total_number
avg_AUROC_train = total_AUROC_train / holdout_total_number
avg_AUROC_test = total_AUROC_test / holdout_total_number
avg_AUPRC_train = total_AUPRC_train / holdout_total_number
avg_AUPRC_test = total_AUPRC_test / holdout_total_number
return {
'accuracy_train': avg_accuracy_train,
'accuracy_test': avg_accuracy_test,
'AUROC_train': avg_AUROC_train,
'AUROC_test': avg_AUROC_test,
'AUPRC_train': avg_AUPRC_train,
'AUPRC_test': avg_AUPRC_test,
}
def plot_train_history(model_history):
""" Plots the metrics (accuracy, AUROC, AUPRC) of the model across the epochs and for each holdout.
Parameters
------------------------
model_history: history
The training record of the model."""
for i in range(len(model_history)):
fig = plt.figure(figsize=(12.7,4.51))
fig.suptitle(str(i+1) + " Holdout")
# Accuracy
plt.subplot(1,3,1)
plt.plot(model_history[i].history['accuracy'])
plt.plot(model_history[i].history['val_accuracy'])
plt.title('Accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
# AUROC
plt.subplot(1,3,2)
plt.plot(model_history[i].history['AUROC'])
plt.plot(model_history[i].history['val_AUROC'])
plt.title('AUROC')
plt.ylabel('AUROC')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
# AUPRC
plt.subplot(1,3,3)
plt.plot(model_history[i].history['AUPRC'])
plt.plot(model_history[i].history['val_AUPRC'])
plt.title('AUPRC')
plt.ylabel('AUPRC')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.subplots_adjust(wspace=0.3)
plt.show()
Hyperparameter tuning is performed training a meta model over training and validation data to obtain the best set of hyperparameters via Bayesian optimization.
Here is the function to train the hypermodel.
def hyperparameter_tuning(
x_train: np.ndarray,
x_validation: np.ndarray,
y_train: np.ndarray,
y_validation: np.ndarray,
hypermodel: tf.keras.Model,
name: str,
directory: str,
max_trials: int = 10,
epochs: int = 50,
batch_size: int = 256
):
""" Tunes the hypermodel with bayesian optiomization and returns the best set of hyperparameters.
Parameters
------------------------
x_train: np.ndarray
The input data for training the hypermodel.
x_validation: np.ndarray
The input data for testing the hypermodel.
y_train: np.ndarray
The labels of the input data for training the hypermodel.
y_validation: np.ndarray
The labels of the input data for testing the hypermodel.
hypermodel: model
The hypermodel to tune or a function that builds a model using hp object.
name: str
The project name.
directory: str
The directory where to store results.
max_trials: int
The maximum amount of model configuration that the algorithm will try out.
epochs: int
The number of times the learning algorithm works through the dataset.
batch_size: int
The number of samples to work through before updating the model.
Returns
------------------------
hyperparameters: hyperparameters
The best set of hyperparameters."""
tuner = kt.tuners.BayesianOptimization(
hypermodel,
objective = [
kt.Objective("val_accuracy", direction="max"),
kt.Objective("val_AUROC", direction="max"),
kt.Objective("val_AUPRC", direction="max")],
max_trials = max_trials,
project_name = name,
directory = directory
)
tuner.search(
x_train,
y_train,
epochs = epochs,
batch_size = batch_size,
validation_data = (x_validation, y_validation),
callbacks = [EarlyStopping("val_loss", patience=2)]
)
best_hp = tuner.get_best_hyperparameters(num_trials=1)[0]
print(f"-- Best set of hyperparameters found:\n{best_hp.values}")
return best_hp
A multi-layer perceptron is a feed forward neural network consiting of one input and output layer with hidden dense layers in-between. To avoid overfitting of the model dropout layers were added.
Here are the functions to build the hypermodel for hyperparameter tuning and to build the model with the best set of hyperparameters found.
def build_MLP_hypermodel(hp) -> tf.keras.Model:
""" Returns the hypermodel.
Parameters
------------------------
hp: hyperparameters
The hyperparameters object.
Returns
------------------------
hypermodel: model
The hyperparamodel."""
MLP_Hypermodel = Sequential(name="MLP_Hypermodel")
MLP_Hypermodel.add(layers.Input((feature_number, )))
for i in range(hp.Int('depth', min_value=1, max_value=5)):
MLP_Hypermodel.add(layers.Dense(units = hp.Int('units_' + str(i), min_value=8, max_value=136, step=32), activation="relu"))
MLP_Hypermodel.add(layers.Dropout(hp.Choice('dropout_'+ str(i), [0.3, 0.4, 0.5])))
MLP_Hypermodel.add(layers.Dense(1, activation='sigmoid'))
MLP_Hypermodel.compile(
optimizer='nadam',
loss='binary_crossentropy',
metrics=get_standard_binary_metrics()
)
MLP_Hypermodel.summary()
return MLP_Hypermodel
def build_MLP(
best_hyperparameters: dict,
input_shape: tuple
) -> tf.keras.Model:
""" Returns the model with the best set of hyperparameters.
Parameters
------------------------
best_hyperparameters: dict
The best set of hyperparameters for the model.
input_shape: tuple
The shape of the input.
Returns
------------------------
model: model
The best model."""
MLP = Sequential(name="MLP")
MLP.add(layers.Input((input_shape[1], )))
for i in range(best_hyperparameters['depth']):
MLP.add(layers.Dense(units = best_hyperparameters['units_' + str(i)], activation="relu"))
MLP.add(layers.Dropout(best_hyperparameters['dropout_' + str(i)]))
MLP.add(layers.Dense(1, activation='sigmoid'))
MLP.compile(
optimizer='nadam',
loss='binary_crossentropy',
metrics=get_standard_binary_metrics()
)
MLP.summary()
return MLP
epochs = 1000
batch_size = 256
print("---- PROMOTERS ----")
MLP_metrics_promoters = {}
MLP_history_promoters = {}
MLP_metrics_promoters["boruta_True"] = []
MLP_metrics_promoters["boruta_False"] = []
MLP_history_promoters["boruta_True"] = []
MLP_history_promoters["boruta_False"] = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(holdouts_generator.split(promoters_epigenomes, promoters_labels)):
print(f"-- HOLDOUT {holdout_number+1} --")
# Train/Test data
x_train_promoters, x_test_promoters = promoters_epigenomes.iloc[train_indices], promoters_epigenomes.iloc[test_indices]
y_train_promoters, y_test_promoters = promoters_labels.iloc[train_indices], promoters_labels.iloc[test_indices]
## Hyperparameter tuning
# Generate holdouts
for holdout_number_tuning, (train_indices_tuning, val_indices_tuning) in enumerate(holdouts_generator_tuning.split(x_train_promoters, y_train_promoters)):
# Train/Validation data
x_train_promoters_tuning, x_val_promoters_tuning = x_train_promoters.iloc[train_indices_tuning], x_train_promoters.iloc[val_indices_tuning]
y_train_promoters_tuning, y_val_promoters_tuning = y_train_promoters.iloc[train_indices_tuning], y_train_promoters.iloc[val_indices_tuning]
hp = kt.HyperParameters()
best_hyperparameters = hyperparameter_tuning(
x_train_promoters_tuning.values,
x_val_promoters_tuning.values,
y_train_promoters_tuning.values,
y_val_promoters_tuning.values,
build_MLP_hypermodel,
name = "MLP_hypermodel_promoters_" + str(holdout_number),
directory='MLP_hypermodel_promoters',
max_trials = 8,
epochs = 50,
batch_size = 256
)
## Remove uncorrelated features with the output
uncorrelated_features = uncorrelated_features_test(x_train_promoters, y_train_promoters)
for feature in (x_train_promoters.columns):
if feature in (uncorrelated_features):
x_train_promoters = x_train_promoters.drop(columns=feature)
x_test_promoters = x_test_promoters.drop(columns=feature)
## Remove correlated features with eachother
correlated_features = correlated_features_test(x_train_promoters)
for feature in (x_train_promoters.columns):
if feature in (correlated_features):
x_train_promoters = x_train_promoters.drop(columns=feature)
x_test_promoters = x_test_promoters.drop(columns=feature)
## Boruta feature selection
for use_Boruta in (False, True):
if use_Boruta:
print("-- Running Boruta Feature Selection:")
kept_features = boruta_feature_selection(x_train_promoters, y_train_promoters)
x_train_promoters = x_train_promoters[kept_features]
x_test_promoters = x_test_promoters[kept_features]
# Build MLP with best set of hyperparameters
MLP = build_MLP(best_hyperparameters, x_train_promoters.shape)
print("- Training model:\n")
MLP_holdout_metrics, MLP_holdout_history = train_model(
MLP,
x_train_promoters.values,
x_test_promoters.values,
y_train_promoters.values,
y_test_promoters.values,
epochs,
batch_size
)
MLP_metrics_promoters["boruta_" + str(use_Boruta)].append(MLP_holdout_metrics)
MLP_history_promoters["boruta_" + str(use_Boruta)].append(MLP_holdout_history)
---- PROMOTERS ----
-- HOLDOUT 1 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_0\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_2 (Dense) (None, 8) 4504
dropout_1 (Dropout) (None, 8) 0
dense_3 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_0\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 72, 'dropout_0': 0.4, 'units_1': 8, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.5, 'units_3': 8, 'dropout_3': 0.3, 'units_4': 104, 'dropout_4': 0.4}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_4 (Dense) (None, 72) 40320
dropout_2 (Dropout) (None, 72) 0
dense_5 (Dense) (None, 1) 73
=================================================================
Total params: 40,393
Trainable params: 40,393
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 10s 23ms/step - loss: 0.4045 - accuracy: 0.8164 - recall: 0.7017 - precision: 0.6295 - AUROC: 0.8674 - AUPRC: 0.6278 - f1_score: 0.6637 - balanced_accuracy: 0.7790 - specificity: 0.8563 - miss_rate: 0.2983 - fall_out: 0.1437 - mcc: 0.5394 - val_loss: 0.3417 - val_accuracy: 0.8411 - val_recall: 0.7559 - val_precision: 0.6705 - val_AUROC: 0.9002 - val_AUPRC: 0.7107 - val_f1_score: 0.7106 - val_balanced_accuracy: 0.8133 - val_specificity: 0.8707 - val_miss_rate: 0.2441 - val_fall_out: 0.1293 - val_mcc: 0.6036
Epoch 2/1000
313/313 [==============================] - 7s 22ms/step - loss: 0.3459 - accuracy: 0.8399 - recall: 0.7345 - precision: 0.6742 - AUROC: 0.8985 - AUPRC: 0.7008 - f1_score: 0.7031 - balanced_accuracy: 0.8055 - specificity: 0.8765 - miss_rate: 0.2655 - fall_out: 0.1235 - mcc: 0.5948 - val_loss: 0.3361 - val_accuracy: 0.8426 - val_recall: 0.7667 - val_precision: 0.6706 - val_AUROC: 0.9044 - val_AUPRC: 0.7156 - val_f1_score: 0.7155 - val_balanced_accuracy: 0.8178 - val_specificity: 0.8690 - val_miss_rate: 0.2333 - val_fall_out: 0.1310 - val_mcc: 0.6099
Epoch 3/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3362 - accuracy: 0.8439 - recall: 0.7414 - precision: 0.6817 - AUROC: 0.9037 - AUPRC: 0.7158 - f1_score: 0.7103 - balanced_accuracy: 0.8105 - specificity: 0.8795 - miss_rate: 0.2586 - fall_out: 0.1205 - mcc: 0.6047 - val_loss: 0.3333 - val_accuracy: 0.8441 - val_recall: 0.7929 - val_precision: 0.6664 - val_AUROC: 0.9064 - val_AUPRC: 0.7274 - val_f1_score: 0.7242 - val_balanced_accuracy: 0.8274 - val_specificity: 0.8619 - val_miss_rate: 0.2071 - val_fall_out: 0.1381 - val_mcc: 0.6211
Epoch 4/1000
313/313 [==============================] - 4s 14ms/step - loss: 0.3277 - accuracy: 0.8471 - recall: 0.7419 - precision: 0.6894 - AUROC: 0.9083 - AUPRC: 0.7290 - f1_score: 0.7147 - balanced_accuracy: 0.8128 - specificity: 0.8837 - miss_rate: 0.2581 - fall_out: 0.1163 - mcc: 0.6112 - val_loss: 0.3283 - val_accuracy: 0.8464 - val_recall: 0.7553 - val_precision: 0.6831 - val_AUROC: 0.9081 - val_AUPRC: 0.7330 - val_f1_score: 0.7174 - val_balanced_accuracy: 0.8167 - val_specificity: 0.8781 - val_miss_rate: 0.2447 - val_fall_out: 0.1219 - val_mcc: 0.6137
Epoch 5/1000
313/313 [==============================] - 6s 18ms/step - loss: 0.3234 - accuracy: 0.8492 - recall: 0.7437 - precision: 0.6940 - AUROC: 0.9109 - AUPRC: 0.7370 - f1_score: 0.7180 - balanced_accuracy: 0.8148 - specificity: 0.8859 - miss_rate: 0.2563 - fall_out: 0.1141 - mcc: 0.6159 - val_loss: 0.3256 - val_accuracy: 0.8474 - val_recall: 0.7448 - val_precision: 0.6891 - val_AUROC: 0.9093 - val_AUPRC: 0.7376 - val_f1_score: 0.7159 - val_balanced_accuracy: 0.8139 - val_specificity: 0.8831 - val_miss_rate: 0.2552 - val_fall_out: 0.1169 - val_mcc: 0.6126
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3200 - accuracy: 0.8520 - recall: 0.7464 - precision: 0.7000 - AUROC: 0.9130 - AUPRC: 0.7413 - f1_score: 0.7225 - balanced_accuracy: 0.8176 - specificity: 0.8887 - miss_rate: 0.2536 - fall_out: 0.1113 - mcc: 0.6223 - val_loss: 0.3283 - val_accuracy: 0.8457 - val_recall: 0.7727 - val_precision: 0.6759 - val_AUROC: 0.9094 - val_AUPRC: 0.7369 - val_f1_score: 0.7211 - val_balanced_accuracy: 0.8219 - val_specificity: 0.8711 - val_miss_rate: 0.2273 - val_fall_out: 0.1289 - val_mcc: 0.6177
Epoch 7/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3170 - accuracy: 0.8526 - recall: 0.7504 - precision: 0.7000 - AUROC: 0.9146 - AUPRC: 0.7465 - f1_score: 0.7243 - balanced_accuracy: 0.8192 - specificity: 0.8881 - miss_rate: 0.2496 - fall_out: 0.1119 - mcc: 0.6245 - val_loss: 0.3248 - val_accuracy: 0.8478 - val_recall: 0.7566 - val_precision: 0.6860 - val_AUROC: 0.9104 - val_AUPRC: 0.7387 - val_f1_score: 0.7196 - val_balanced_accuracy: 0.8181 - val_specificity: 0.8795 - val_miss_rate: 0.2434 - val_fall_out: 0.1205 - val_mcc: 0.6169
Epoch 8/1000
313/313 [==============================] - 6s 18ms/step - loss: 0.3137 - accuracy: 0.8556 - recall: 0.7506 - precision: 0.7076 - AUROC: 0.9167 - AUPRC: 0.7520 - f1_score: 0.7285 - balanced_accuracy: 0.8213 - specificity: 0.8921 - miss_rate: 0.2494 - fall_out: 0.1079 - mcc: 0.6307 - val_loss: 0.3270 - val_accuracy: 0.8456 - val_recall: 0.6940 - val_precision: 0.7038 - val_AUROC: 0.9094 - val_AUPRC: 0.7389 - val_f1_score: 0.6989 - val_balanced_accuracy: 0.7962 - val_specificity: 0.8984 - val_miss_rate: 0.3060 - val_fall_out: 0.1016 - val_mcc: 0.5951
Epoch 9/1000
313/313 [==============================] - 5s 16ms/step - loss: 0.3116 - accuracy: 0.8569 - recall: 0.7526 - precision: 0.7104 - AUROC: 0.9182 - AUPRC: 0.7567 - f1_score: 0.7309 - balanced_accuracy: 0.8229 - specificity: 0.8932 - miss_rate: 0.2474 - fall_out: 0.1068 - mcc: 0.6341 - val_loss: 0.3475 - val_accuracy: 0.8393 - val_recall: 0.6907 - val_precision: 0.6880 - val_AUROC: 0.8964 - val_AUPRC: 0.7148 - val_f1_score: 0.6894 - val_balanced_accuracy: 0.7909 - val_specificity: 0.8910 - val_miss_rate: 0.3093 - val_fall_out: 0.1090 - val_mcc: 0.5810
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 8 / 100
Confirmed: 400
Tentative: 159
Rejected: 0
Iteration: 9 / 100
Confirmed: 400
Tentative: 159
Rejected: 0
Iteration: 10 / 100
Confirmed: 400
Tentative: 159
Rejected: 0
Iteration: 11 / 100
Confirmed: 400
Tentative: 159
Rejected: 0
Iteration: 12 / 100
Confirmed: 434
Tentative: 125
Rejected: 0
Iteration: 13 / 100
Confirmed: 434
Tentative: 125
Rejected: 0
Iteration: 14 / 100
Confirmed: 434
Tentative: 124
Rejected: 1
Iteration: 15 / 100
Confirmed: 434
Tentative: 124
Rejected: 1
Iteration: 16 / 100
Confirmed: 444
Tentative: 114
Rejected: 1
Iteration: 17 / 100
Confirmed: 444
Tentative: 114
Rejected: 1
Iteration: 18 / 100
Confirmed: 444
Tentative: 114
Rejected: 1
Iteration: 19 / 100
Confirmed: 451
Tentative: 107
Rejected: 1
Iteration: 20 / 100
Confirmed: 451
Tentative: 104
Rejected: 4
Iteration: 21 / 100
Confirmed: 451
Tentative: 104
Rejected: 4
Iteration: 22 / 100
Confirmed: 457
Tentative: 98
Rejected: 4
Iteration: 23 / 100
Confirmed: 457
Tentative: 98
Rejected: 4
Iteration: 24 / 100
Confirmed: 457
Tentative: 98
Rejected: 4
Iteration: 25 / 100
Confirmed: 457
Tentative: 98
Rejected: 4
Iteration: 26 / 100
Confirmed: 462
Tentative: 93
Rejected: 4
Iteration: 27 / 100
Confirmed: 462
Tentative: 91
Rejected: 6
Iteration: 28 / 100
Confirmed: 462
Tentative: 91
Rejected: 6
Iteration: 29 / 100
Confirmed: 466
Tentative: 87
Rejected: 6
Iteration: 30 / 100
Confirmed: 466
Tentative: 87
Rejected: 6
Iteration: 31 / 100
Confirmed: 466
Tentative: 87
Rejected: 6
Iteration: 32 / 100
Confirmed: 471
Tentative: 81
Rejected: 7
Iteration: 33 / 100
Confirmed: 471
Tentative: 81
Rejected: 7
Iteration: 34 / 100
Confirmed: 478
Tentative: 74
Rejected: 7
Iteration: 35 / 100
Confirmed: 478
Tentative: 74
Rejected: 7
Iteration: 36 / 100
Confirmed: 478
Tentative: 74
Rejected: 7
Iteration: 37 / 100
Confirmed: 481
Tentative: 71
Rejected: 7
Iteration: 38 / 100
Confirmed: 481
Tentative: 71
Rejected: 7
Iteration: 39 / 100
Confirmed: 481
Tentative: 71
Rejected: 7
Iteration: 40 / 100
Confirmed: 483
Tentative: 69
Rejected: 7
Iteration: 41 / 100
Confirmed: 483
Tentative: 68
Rejected: 8
Iteration: 42 / 100
Confirmed: 483
Tentative: 68
Rejected: 8
Iteration: 43 / 100
Confirmed: 485
Tentative: 66
Rejected: 8
Iteration: 44 / 100
Confirmed: 485
Tentative: 66
Rejected: 8
Iteration: 45 / 100
Confirmed: 485
Tentative: 66
Rejected: 8
Iteration: 46 / 100
Confirmed: 486
Tentative: 62
Rejected: 11
Iteration: 47 / 100
Confirmed: 486
Tentative: 62
Rejected: 11
Iteration: 48 / 100
Confirmed: 486
Tentative: 62
Rejected: 11
Iteration: 49 / 100
Confirmed: 487
Tentative: 61
Rejected: 11
Iteration: 50 / 100
Confirmed: 487
Tentative: 61
Rejected: 11
Iteration: 51 / 100
Confirmed: 487
Tentative: 61
Rejected: 11
Iteration: 52 / 100
Confirmed: 487
Tentative: 61
Rejected: 11
Iteration: 53 / 100
Confirmed: 487
Tentative: 61
Rejected: 11
Iteration: 54 / 100
Confirmed: 487
Tentative: 57
Rejected: 15
Iteration: 55 / 100
Confirmed: 487
Tentative: 57
Rejected: 15
Iteration: 56 / 100
Confirmed: 487
Tentative: 57
Rejected: 15
Iteration: 57 / 100
Confirmed: 488
Tentative: 56
Rejected: 15
Iteration: 58 / 100
Confirmed: 488
Tentative: 56
Rejected: 15
Iteration: 59 / 100
Confirmed: 488
Tentative: 56
Rejected: 15
Iteration: 60 / 100
Confirmed: 488
Tentative: 56
Rejected: 15
Iteration: 61 / 100
Confirmed: 488
Tentative: 56
Rejected: 15
Iteration: 62 / 100
Confirmed: 488
Tentative: 56
Rejected: 15
Iteration: 63 / 100
Confirmed: 488
Tentative: 56
Rejected: 15
Iteration: 64 / 100
Confirmed: 488
Tentative: 56
Rejected: 15
Iteration: 65 / 100
Confirmed: 489
Tentative: 55
Rejected: 15
Iteration: 66 / 100
Confirmed: 489
Tentative: 55
Rejected: 15
Iteration: 67 / 100
Confirmed: 490
Tentative: 54
Rejected: 15
Iteration: 68 / 100
Confirmed: 490
Tentative: 52
Rejected: 17
Iteration: 69 / 100
Confirmed: 490
Tentative: 52
Rejected: 17
Iteration: 70 / 100
Confirmed: 491
Tentative: 51
Rejected: 17
Iteration: 71 / 100
Confirmed: 491
Tentative: 51
Rejected: 17
Iteration: 72 / 100
Confirmed: 491
Tentative: 51
Rejected: 17
Iteration: 73 / 100
Confirmed: 491
Tentative: 51
Rejected: 17
Iteration: 74 / 100
Confirmed: 491
Tentative: 49
Rejected: 19
Iteration: 75 / 100
Confirmed: 493
Tentative: 47
Rejected: 19
Iteration: 76 / 100
Confirmed: 493
Tentative: 47
Rejected: 19
Iteration: 77 / 100
Confirmed: 493
Tentative: 47
Rejected: 19
Iteration: 78 / 100
Confirmed: 493
Tentative: 47
Rejected: 19
Iteration: 79 / 100
Confirmed: 493
Tentative: 47
Rejected: 19
Iteration: 80 / 100
Confirmed: 494
Tentative: 46
Rejected: 19
Iteration: 81 / 100
Confirmed: 494
Tentative: 46
Rejected: 19
Iteration: 82 / 100
Confirmed: 494
Tentative: 43
Rejected: 22
Iteration: 83 / 100
Confirmed: 494
Tentative: 43
Rejected: 22
Iteration: 84 / 100
Confirmed: 494
Tentative: 43
Rejected: 22
Iteration: 85 / 100
Confirmed: 495
Tentative: 42
Rejected: 22
Iteration: 86 / 100
Confirmed: 495
Tentative: 42
Rejected: 22
Iteration: 87 / 100
Confirmed: 495
Tentative: 42
Rejected: 22
Iteration: 88 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 89 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 90 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 91 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 92 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 93 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 94 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 95 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 96 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 97 / 100
Confirmed: 495
Tentative: 41
Rejected: 23
Iteration: 98 / 100
Confirmed: 495
Tentative: 40
Rejected: 24
Iteration: 99 / 100
Confirmed: 495
Tentative: 40
Rejected: 24
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 495
Tentative: 15
Rejected: 24
-- 495 Kept (correlated) features [Boruta]
-- 64 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_6 (Dense) (None, 72) 35712
dropout_3 (Dropout) (None, 72) 0
dense_7 (Dense) (None, 1) 73
=================================================================
Total params: 35,785
Trainable params: 35,785
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 8s 21ms/step - loss: 0.4017 - accuracy: 0.8192 - recall: 0.6829 - precision: 0.6405 - AUROC: 0.8717 - AUPRC: 0.6424 - f1_score: 0.6610 - balanced_accuracy: 0.7748 - specificity: 0.8666 - miss_rate: 0.3171 - fall_out: 0.1334 - mcc: 0.5385 - val_loss: 0.3483 - val_accuracy: 0.8343 - val_recall: 0.6711 - val_precision: 0.6820 - val_AUROC: 0.8963 - val_AUPRC: 0.6961 - val_f1_score: 0.6765 - val_balanced_accuracy: 0.7811 - val_specificity: 0.8911 - val_miss_rate: 0.3289 - val_fall_out: 0.1089 - val_mcc: 0.5652
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3455 - accuracy: 0.8394 - recall: 0.7276 - precision: 0.6754 - AUROC: 0.8979 - AUPRC: 0.6984 - f1_score: 0.7005 - balanced_accuracy: 0.8030 - specificity: 0.8783 - miss_rate: 0.2724 - fall_out: 0.1217 - mcc: 0.5918 - val_loss: 0.3394 - val_accuracy: 0.8410 - val_recall: 0.6890 - val_precision: 0.6931 - val_AUROC: 0.9023 - val_AUPRC: 0.7202 - val_f1_score: 0.6910 - val_balanced_accuracy: 0.7914 - val_specificity: 0.8939 - val_miss_rate: 0.3110 - val_fall_out: 0.1061 - val_mcc: 0.5840
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3344 - accuracy: 0.8446 - recall: 0.7323 - precision: 0.6866 - AUROC: 0.9043 - AUPRC: 0.7160 - f1_score: 0.7087 - balanced_accuracy: 0.8080 - specificity: 0.8837 - miss_rate: 0.2677 - fall_out: 0.1163 - mcc: 0.6035 - val_loss: 0.3361 - val_accuracy: 0.8416 - val_recall: 0.7882 - val_precision: 0.6624 - val_AUROC: 0.9051 - val_AUPRC: 0.7222 - val_f1_score: 0.7199 - val_balanced_accuracy: 0.8242 - val_specificity: 0.8602 - val_miss_rate: 0.2118 - val_fall_out: 0.1398 - val_mcc: 0.6151
Epoch 4/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3280 - accuracy: 0.8473 - recall: 0.7384 - precision: 0.6911 - AUROC: 0.9081 - AUPRC: 0.7260 - f1_score: 0.7140 - balanced_accuracy: 0.8118 - specificity: 0.8852 - miss_rate: 0.2616 - fall_out: 0.1148 - mcc: 0.6106 - val_loss: 0.3330 - val_accuracy: 0.8440 - val_recall: 0.7111 - val_precision: 0.6927 - val_AUROC: 0.9051 - val_AUPRC: 0.7241 - val_f1_score: 0.7018 - val_balanced_accuracy: 0.8006 - val_specificity: 0.8902 - val_miss_rate: 0.2889 - val_fall_out: 0.1098 - val_mcc: 0.5962
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3241 - accuracy: 0.8488 - recall: 0.7397 - precision: 0.6944 - AUROC: 0.9103 - AUPRC: 0.7327 - f1_score: 0.7163 - balanced_accuracy: 0.8132 - specificity: 0.8867 - miss_rate: 0.2603 - fall_out: 0.1133 - mcc: 0.6139 - val_loss: 0.3346 - val_accuracy: 0.8421 - val_recall: 0.6387 - val_precision: 0.7184 - val_AUROC: 0.9064 - val_AUPRC: 0.7321 - val_f1_score: 0.6762 - val_balanced_accuracy: 0.7758 - val_specificity: 0.9129 - val_miss_rate: 0.3613 - val_fall_out: 0.0871 - val_mcc: 0.5741
Epoch 6/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3211 - accuracy: 0.8513 - recall: 0.7412 - precision: 0.7002 - AUROC: 0.9123 - AUPRC: 0.7391 - f1_score: 0.7201 - balanced_accuracy: 0.8154 - specificity: 0.8896 - miss_rate: 0.2588 - fall_out: 0.1104 - mcc: 0.6194 - val_loss: 0.3273 - val_accuracy: 0.8454 - val_recall: 0.7374 - val_precision: 0.6868 - val_AUROC: 0.9085 - val_AUPRC: 0.7360 - val_f1_score: 0.7112 - val_balanced_accuracy: 0.8102 - val_specificity: 0.8830 - val_miss_rate: 0.2626 - val_fall_out: 0.1170 - val_mcc: 0.6066
Epoch 7/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3174 - accuracy: 0.8526 - recall: 0.7434 - precision: 0.7027 - AUROC: 0.9145 - AUPRC: 0.7452 - f1_score: 0.7225 - balanced_accuracy: 0.8170 - specificity: 0.8906 - miss_rate: 0.2566 - fall_out: 0.1094 - mcc: 0.6227 - val_loss: 0.3271 - val_accuracy: 0.8468 - val_recall: 0.7677 - val_precision: 0.6800 - val_AUROC: 0.9094 - val_AUPRC: 0.7390 - val_f1_score: 0.7212 - val_balanced_accuracy: 0.8210 - val_specificity: 0.8743 - val_miss_rate: 0.2323 - val_fall_out: 0.1257 - val_mcc: 0.6182
Epoch 8/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3149 - accuracy: 0.8545 - recall: 0.7526 - precision: 0.7041 - AUROC: 0.9158 - AUPRC: 0.7516 - f1_score: 0.7275 - balanced_accuracy: 0.8213 - specificity: 0.8899 - miss_rate: 0.2474 - fall_out: 0.1101 - mcc: 0.6291 - val_loss: 0.3252 - val_accuracy: 0.8471 - val_recall: 0.7198 - val_precision: 0.6976 - val_AUROC: 0.9099 - val_AUPRC: 0.7398 - val_f1_score: 0.7085 - val_balanced_accuracy: 0.8056 - val_specificity: 0.8914 - val_miss_rate: 0.2802 - val_fall_out: 0.1086 - val_mcc: 0.6051
Epoch 9/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3120 - accuracy: 0.8560 - recall: 0.7524 - precision: 0.7079 - AUROC: 0.9175 - AUPRC: 0.7564 - f1_score: 0.7295 - balanced_accuracy: 0.8222 - specificity: 0.8920 - miss_rate: 0.2476 - fall_out: 0.1080 - mcc: 0.6320 - val_loss: 0.3239 - val_accuracy: 0.8473 - val_recall: 0.7469 - val_precision: 0.6881 - val_AUROC: 0.9106 - val_AUPRC: 0.7406 - val_f1_score: 0.7163 - val_balanced_accuracy: 0.8146 - val_specificity: 0.8822 - val_miss_rate: 0.2531 - val_fall_out: 0.1178 - val_mcc: 0.6130
Epoch 10/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3096 - accuracy: 0.8558 - recall: 0.7527 - precision: 0.7073 - AUROC: 0.9189 - AUPRC: 0.7594 - f1_score: 0.7293 - balanced_accuracy: 0.8222 - specificity: 0.8916 - miss_rate: 0.2473 - fall_out: 0.1084 - mcc: 0.6317 - val_loss: 0.3239 - val_accuracy: 0.8481 - val_recall: 0.7817 - val_precision: 0.6786 - val_AUROC: 0.9119 - val_AUPRC: 0.7447 - val_f1_score: 0.7265 - val_balanced_accuracy: 0.8264 - val_specificity: 0.8712 - val_miss_rate: 0.2183 - val_fall_out: 0.1288 - val_mcc: 0.6250
Epoch 11/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3075 - accuracy: 0.8587 - recall: 0.7546 - precision: 0.7141 - AUROC: 0.9202 - AUPRC: 0.7628 - f1_score: 0.7338 - balanced_accuracy: 0.8247 - specificity: 0.8949 - miss_rate: 0.2454 - fall_out: 0.1051 - mcc: 0.6381 - val_loss: 0.3293 - val_accuracy: 0.8443 - val_recall: 0.7886 - val_precision: 0.6681 - val_AUROC: 0.9091 - val_AUPRC: 0.7348 - val_f1_score: 0.7234 - val_balanced_accuracy: 0.8262 - val_specificity: 0.8637 - val_miss_rate: 0.2114 - val_fall_out: 0.1363 - val_mcc: 0.6202
Epoch 12/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3060 - accuracy: 0.8597 - recall: 0.7592 - precision: 0.7148 - AUROC: 0.9213 - AUPRC: 0.7672 - f1_score: 0.7363 - balanced_accuracy: 0.8269 - specificity: 0.8946 - miss_rate: 0.2408 - fall_out: 0.1054 - mcc: 0.6414 - val_loss: 0.3253 - val_accuracy: 0.8478 - val_recall: 0.7268 - val_precision: 0.6967 - val_AUROC: 0.9101 - val_AUPRC: 0.7356 - val_f1_score: 0.7114 - val_balanced_accuracy: 0.8083 - val_specificity: 0.8899 - val_miss_rate: 0.2732 - val_fall_out: 0.1101 - val_mcc: 0.6083
-- HOLDOUT 2 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_1\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_8 (Dense) (None, 8) 4504
dropout_4 (Dropout) (None, 8) 0
dense_9 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_1\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 136, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.3, 'units_4': 40, 'dropout_4': 0.5}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_10 (Dense) (None, 136) 76160
dropout_5 (Dropout) (None, 136) 0
dense_11 (Dense) (None, 1) 137
=================================================================
Total params: 76,297
Trainable params: 76,297
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.4153 - accuracy: 0.8126 - recall: 0.6824 - precision: 0.6255 - AUROC: 0.8646 - AUPRC: 0.6275 - f1_score: 0.6527 - balanced_accuracy: 0.7701 - specificity: 0.8579 - miss_rate: 0.3176 - fall_out: 0.1421 - mcc: 0.5257 - val_loss: 0.3456 - val_accuracy: 0.8392 - val_recall: 0.7018 - val_precision: 0.6836 - val_AUROC: 0.8994 - val_AUPRC: 0.7062 - val_f1_score: 0.6926 - val_balanced_accuracy: 0.7944 - val_specificity: 0.8870 - val_miss_rate: 0.2982 - val_fall_out: 0.1130 - val_mcc: 0.5838
Epoch 2/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3469 - accuracy: 0.8387 - recall: 0.7312 - precision: 0.6724 - AUROC: 0.8979 - AUPRC: 0.7001 - f1_score: 0.7006 - balanced_accuracy: 0.8036 - specificity: 0.8761 - miss_rate: 0.2688 - fall_out: 0.1239 - mcc: 0.5914 - val_loss: 0.3388 - val_accuracy: 0.8434 - val_recall: 0.7126 - val_precision: 0.6905 - val_AUROC: 0.9034 - val_AUPRC: 0.7167 - val_f1_score: 0.7014 - val_balanced_accuracy: 0.8007 - val_specificity: 0.8889 - val_miss_rate: 0.2874 - val_fall_out: 0.1111 - val_mcc: 0.5954
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3369 - accuracy: 0.8439 - recall: 0.7343 - precision: 0.6841 - AUROC: 0.9035 - AUPRC: 0.7168 - f1_score: 0.7083 - balanced_accuracy: 0.8082 - specificity: 0.8820 - miss_rate: 0.2657 - fall_out: 0.1180 - mcc: 0.6026 - val_loss: 0.3296 - val_accuracy: 0.8469 - val_recall: 0.7640 - val_precision: 0.6814 - val_AUROC: 0.9082 - val_AUPRC: 0.7297 - val_f1_score: 0.7204 - val_balanced_accuracy: 0.8199 - val_specificity: 0.8757 - val_miss_rate: 0.2360 - val_fall_out: 0.1243 - val_mcc: 0.6173
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3299 - accuracy: 0.8471 - recall: 0.7429 - precision: 0.6889 - AUROC: 0.9077 - AUPRC: 0.7283 - f1_score: 0.7149 - balanced_accuracy: 0.8131 - specificity: 0.8833 - miss_rate: 0.2571 - fall_out: 0.1167 - mcc: 0.6114 - val_loss: 0.3294 - val_accuracy: 0.8465 - val_recall: 0.7708 - val_precision: 0.6783 - val_AUROC: 0.9086 - val_AUPRC: 0.7293 - val_f1_score: 0.7216 - val_balanced_accuracy: 0.8218 - val_specificity: 0.8728 - val_miss_rate: 0.2292 - val_fall_out: 0.1272 - val_mcc: 0.6186
Epoch 5/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3257 - accuracy: 0.8497 - recall: 0.7429 - precision: 0.6956 - AUROC: 0.9098 - AUPRC: 0.7356 - f1_score: 0.7185 - balanced_accuracy: 0.8149 - specificity: 0.8869 - miss_rate: 0.2571 - fall_out: 0.1131 - mcc: 0.6168 - val_loss: 0.3264 - val_accuracy: 0.8478 - val_recall: 0.7710 - val_precision: 0.6813 - val_AUROC: 0.9097 - val_AUPRC: 0.7301 - val_f1_score: 0.7234 - val_balanced_accuracy: 0.8227 - val_specificity: 0.8745 - val_miss_rate: 0.2290 - val_fall_out: 0.1255 - val_mcc: 0.6212
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3220 - accuracy: 0.8510 - recall: 0.7449 - precision: 0.6981 - AUROC: 0.9121 - AUPRC: 0.7426 - f1_score: 0.7207 - balanced_accuracy: 0.8164 - specificity: 0.8879 - miss_rate: 0.2551 - fall_out: 0.1121 - mcc: 0.6199 - val_loss: 0.3261 - val_accuracy: 0.8487 - val_recall: 0.7739 - val_precision: 0.6826 - val_AUROC: 0.9106 - val_AUPRC: 0.7322 - val_f1_score: 0.7254 - val_balanced_accuracy: 0.8243 - val_specificity: 0.8748 - val_miss_rate: 0.2261 - val_fall_out: 0.1252 - val_mcc: 0.6239
Epoch 7/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3199 - accuracy: 0.8526 - recall: 0.7523 - precision: 0.6993 - AUROC: 0.9135 - AUPRC: 0.7456 - f1_score: 0.7248 - balanced_accuracy: 0.8199 - specificity: 0.8875 - miss_rate: 0.2477 - fall_out: 0.1125 - mcc: 0.6251 - val_loss: 0.3229 - val_accuracy: 0.8508 - val_recall: 0.7076 - val_precision: 0.7124 - val_AUROC: 0.9124 - val_AUPRC: 0.7402 - val_f1_score: 0.7100 - val_balanced_accuracy: 0.8041 - val_specificity: 0.9006 - val_miss_rate: 0.2924 - val_fall_out: 0.0994 - val_mcc: 0.6095
Epoch 8/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3157 - accuracy: 0.8539 - recall: 0.7488 - precision: 0.7041 - AUROC: 0.9156 - AUPRC: 0.7522 - f1_score: 0.7258 - balanced_accuracy: 0.8196 - specificity: 0.8905 - miss_rate: 0.2512 - fall_out: 0.1095 - mcc: 0.6269 - val_loss: 0.3220 - val_accuracy: 0.8509 - val_recall: 0.7671 - val_precision: 0.6900 - val_AUROC: 0.9123 - val_AUPRC: 0.7379 - val_f1_score: 0.7265 - val_balanced_accuracy: 0.8236 - val_specificity: 0.8801 - val_miss_rate: 0.2329 - val_fall_out: 0.1199 - val_mcc: 0.6261
Epoch 9/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3138 - accuracy: 0.8552 - recall: 0.7494 - precision: 0.7072 - AUROC: 0.9169 - AUPRC: 0.7547 - f1_score: 0.7277 - balanced_accuracy: 0.8207 - specificity: 0.8921 - miss_rate: 0.2506 - fall_out: 0.1079 - mcc: 0.6297 - val_loss: 0.3221 - val_accuracy: 0.8504 - val_recall: 0.7555 - val_precision: 0.6929 - val_AUROC: 0.9126 - val_AUPRC: 0.7382 - val_f1_score: 0.7228 - val_balanced_accuracy: 0.8195 - val_specificity: 0.8835 - val_miss_rate: 0.2445 - val_fall_out: 0.1165 - val_mcc: 0.6218
Epoch 10/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3106 - accuracy: 0.8578 - recall: 0.7510 - precision: 0.7133 - AUROC: 0.9185 - AUPRC: 0.7617 - f1_score: 0.7316 - balanced_accuracy: 0.8230 - specificity: 0.8950 - miss_rate: 0.2490 - fall_out: 0.1050 - mcc: 0.6354 - val_loss: 0.3307 - val_accuracy: 0.8467 - val_recall: 0.8156 - val_precision: 0.6658 - val_AUROC: 0.9123 - val_AUPRC: 0.7368 - val_f1_score: 0.7331 - val_balanced_accuracy: 0.8366 - val_specificity: 0.8576 - val_miss_rate: 0.1844 - val_fall_out: 0.1424 - val_mcc: 0.6335
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 8 / 100
Confirmed: 396
Tentative: 163
Rejected: 0
Iteration: 9 / 100
Confirmed: 396
Tentative: 163
Rejected: 0
Iteration: 10 / 100
Confirmed: 396
Tentative: 163
Rejected: 0
Iteration: 11 / 100
Confirmed: 396
Tentative: 163
Rejected: 0
Iteration: 12 / 100
Confirmed: 424
Tentative: 131
Rejected: 4
Iteration: 13 / 100
Confirmed: 424
Tentative: 131
Rejected: 4
Iteration: 14 / 100
Confirmed: 424
Tentative: 131
Rejected: 4
Iteration: 15 / 100
Confirmed: 424
Tentative: 131
Rejected: 4
Iteration: 16 / 100
Confirmed: 441
Tentative: 114
Rejected: 4
Iteration: 17 / 100
Confirmed: 441
Tentative: 114
Rejected: 4
Iteration: 18 / 100
Confirmed: 441
Tentative: 114
Rejected: 4
Iteration: 19 / 100
Confirmed: 451
Tentative: 104
Rejected: 4
Iteration: 20 / 100
Confirmed: 451
Tentative: 104
Rejected: 4
Iteration: 21 / 100
Confirmed: 451
Tentative: 104
Rejected: 4
Iteration: 22 / 100
Confirmed: 454
Tentative: 101
Rejected: 4
Iteration: 23 / 100
Confirmed: 454
Tentative: 97
Rejected: 8
Iteration: 24 / 100
Confirmed: 454
Tentative: 97
Rejected: 8
Iteration: 25 / 100
Confirmed: 454
Tentative: 97
Rejected: 8
Iteration: 26 / 100
Confirmed: 456
Tentative: 95
Rejected: 8
Iteration: 27 / 100
Confirmed: 456
Tentative: 95
Rejected: 8
Iteration: 28 / 100
Confirmed: 456
Tentative: 94
Rejected: 9
Iteration: 29 / 100
Confirmed: 461
Tentative: 89
Rejected: 9
Iteration: 30 / 100
Confirmed: 461
Tentative: 89
Rejected: 9
Iteration: 31 / 100
Confirmed: 461
Tentative: 89
Rejected: 9
Iteration: 32 / 100
Confirmed: 465
Tentative: 85
Rejected: 9
Iteration: 33 / 100
Confirmed: 465
Tentative: 85
Rejected: 9
Iteration: 34 / 100
Confirmed: 465
Tentative: 85
Rejected: 9
Iteration: 35 / 100
Confirmed: 465
Tentative: 84
Rejected: 10
Iteration: 36 / 100
Confirmed: 465
Tentative: 84
Rejected: 10
Iteration: 37 / 100
Confirmed: 466
Tentative: 83
Rejected: 10
Iteration: 38 / 100
Confirmed: 466
Tentative: 80
Rejected: 13
Iteration: 39 / 100
Confirmed: 466
Tentative: 80
Rejected: 13
Iteration: 40 / 100
Confirmed: 469
Tentative: 77
Rejected: 13
Iteration: 41 / 100
Confirmed: 469
Tentative: 77
Rejected: 13
Iteration: 42 / 100
Confirmed: 469
Tentative: 77
Rejected: 13
Iteration: 43 / 100
Confirmed: 470
Tentative: 76
Rejected: 13
Iteration: 44 / 100
Confirmed: 470
Tentative: 76
Rejected: 13
Iteration: 45 / 100
Confirmed: 470
Tentative: 76
Rejected: 13
Iteration: 46 / 100
Confirmed: 470
Tentative: 76
Rejected: 13
Iteration: 47 / 100
Confirmed: 470
Tentative: 76
Rejected: 13
Iteration: 48 / 100
Confirmed: 470
Tentative: 76
Rejected: 13
Iteration: 49 / 100
Confirmed: 470
Tentative: 75
Rejected: 14
Iteration: 50 / 100
Confirmed: 470
Tentative: 75
Rejected: 14
Iteration: 51 / 100
Confirmed: 471
Tentative: 74
Rejected: 14
Iteration: 52 / 100
Confirmed: 471
Tentative: 74
Rejected: 14
Iteration: 53 / 100
Confirmed: 471
Tentative: 74
Rejected: 14
Iteration: 54 / 100
Confirmed: 471
Tentative: 74
Rejected: 14
Iteration: 55 / 100
Confirmed: 471
Tentative: 74
Rejected: 14
Iteration: 56 / 100
Confirmed: 471
Tentative: 74
Rejected: 14
Iteration: 57 / 100
Confirmed: 472
Tentative: 73
Rejected: 14
Iteration: 58 / 100
Confirmed: 472
Tentative: 73
Rejected: 14
Iteration: 59 / 100
Confirmed: 474
Tentative: 71
Rejected: 14
Iteration: 60 / 100
Confirmed: 474
Tentative: 71
Rejected: 14
Iteration: 61 / 100
Confirmed: 474
Tentative: 67
Rejected: 18
Iteration: 62 / 100
Confirmed: 474
Tentative: 67
Rejected: 18
Iteration: 63 / 100
Confirmed: 474
Tentative: 67
Rejected: 18
Iteration: 64 / 100
Confirmed: 474
Tentative: 67
Rejected: 18
Iteration: 65 / 100
Confirmed: 475
Tentative: 66
Rejected: 18
Iteration: 66 / 100
Confirmed: 475
Tentative: 66
Rejected: 18
Iteration: 67 / 100
Confirmed: 475
Tentative: 63
Rejected: 21
Iteration: 68 / 100
Confirmed: 475
Tentative: 63
Rejected: 21
Iteration: 69 / 100
Confirmed: 475
Tentative: 63
Rejected: 21
Iteration: 70 / 100
Confirmed: 475
Tentative: 63
Rejected: 21
Iteration: 71 / 100
Confirmed: 475
Tentative: 63
Rejected: 21
Iteration: 72 / 100
Confirmed: 477
Tentative: 61
Rejected: 21
Iteration: 73 / 100
Confirmed: 477
Tentative: 61
Rejected: 21
Iteration: 74 / 100
Confirmed: 477
Tentative: 61
Rejected: 21
Iteration: 75 / 100
Confirmed: 477
Tentative: 60
Rejected: 22
Iteration: 76 / 100
Confirmed: 477
Tentative: 60
Rejected: 22
Iteration: 77 / 100
Confirmed: 478
Tentative: 59
Rejected: 22
Iteration: 78 / 100
Confirmed: 478
Tentative: 58
Rejected: 23
Iteration: 79 / 100
Confirmed: 478
Tentative: 58
Rejected: 23
Iteration: 80 / 100
Confirmed: 478
Tentative: 57
Rejected: 24
Iteration: 81 / 100
Confirmed: 478
Tentative: 57
Rejected: 24
Iteration: 82 / 100
Confirmed: 478
Tentative: 57
Rejected: 24
Iteration: 83 / 100
Confirmed: 479
Tentative: 56
Rejected: 24
Iteration: 84 / 100
Confirmed: 479
Tentative: 56
Rejected: 24
Iteration: 85 / 100
Confirmed: 479
Tentative: 56
Rejected: 24
Iteration: 86 / 100
Confirmed: 479
Tentative: 56
Rejected: 24
Iteration: 87 / 100
Confirmed: 479
Tentative: 56
Rejected: 24
Iteration: 88 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 89 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 90 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 91 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 92 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 93 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 94 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 95 / 100
Confirmed: 483
Tentative: 52
Rejected: 24
Iteration: 96 / 100
Confirmed: 483
Tentative: 52
Rejected: 24
Iteration: 97 / 100
Confirmed: 483
Tentative: 52
Rejected: 24
Iteration: 98 / 100
Confirmed: 484
Tentative: 50
Rejected: 25
Iteration: 99 / 100
Confirmed: 484
Tentative: 50
Rejected: 25
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 484
Tentative: 25
Rejected: 25
-- 484 Kept (correlated) features [Boruta]
-- 75 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_12 (Dense) (None, 136) 65960
dropout_6 (Dropout) (None, 136) 0
dense_13 (Dense) (None, 1) 137
=================================================================
Total params: 66,097
Trainable params: 66,097
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.4026 - accuracy: 0.8194 - recall: 0.6858 - precision: 0.6402 - AUROC: 0.8718 - AUPRC: 0.6417 - f1_score: 0.6622 - balanced_accuracy: 0.7758 - specificity: 0.8659 - miss_rate: 0.3142 - fall_out: 0.1341 - mcc: 0.5398 - val_loss: 0.3413 - val_accuracy: 0.8421 - val_recall: 0.7632 - val_precision: 0.6706 - val_AUROC: 0.9015 - val_AUPRC: 0.7104 - val_f1_score: 0.7139 - val_balanced_accuracy: 0.8164 - val_specificity: 0.8696 - val_miss_rate: 0.2368 - val_fall_out: 0.1304 - val_mcc: 0.6080
Epoch 2/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3462 - accuracy: 0.8391 - recall: 0.7296 - precision: 0.6739 - AUROC: 0.8983 - AUPRC: 0.7000 - f1_score: 0.7007 - balanced_accuracy: 0.8034 - specificity: 0.8772 - miss_rate: 0.2704 - fall_out: 0.1228 - mcc: 0.5917 - val_loss: 0.3339 - val_accuracy: 0.8457 - val_recall: 0.7285 - val_precision: 0.6906 - val_AUROC: 0.9055 - val_AUPRC: 0.7243 - val_f1_score: 0.7091 - val_balanced_accuracy: 0.8075 - val_specificity: 0.8864 - val_miss_rate: 0.2715 - val_fall_out: 0.1136 - val_mcc: 0.6046
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3346 - accuracy: 0.8438 - recall: 0.7379 - precision: 0.6825 - AUROC: 0.9046 - AUPRC: 0.7160 - f1_score: 0.7091 - balanced_accuracy: 0.8093 - specificity: 0.8806 - miss_rate: 0.2621 - fall_out: 0.1194 - mcc: 0.6034 - val_loss: 0.3323 - val_accuracy: 0.8434 - val_recall: 0.6583 - val_precision: 0.7131 - val_AUROC: 0.9082 - val_AUPRC: 0.7336 - val_f1_score: 0.6846 - val_balanced_accuracy: 0.7831 - val_specificity: 0.9078 - val_miss_rate: 0.3417 - val_fall_out: 0.0922 - val_mcc: 0.5815
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3285 - accuracy: 0.8467 - recall: 0.7402 - precision: 0.6889 - AUROC: 0.9080 - AUPRC: 0.7312 - f1_score: 0.7136 - balanced_accuracy: 0.8120 - specificity: 0.8837 - miss_rate: 0.2598 - fall_out: 0.1163 - mcc: 0.6098 - val_loss: 0.3269 - val_accuracy: 0.8488 - val_recall: 0.7471 - val_precision: 0.6919 - val_AUROC: 0.9096 - val_AUPRC: 0.7306 - val_f1_score: 0.7184 - val_balanced_accuracy: 0.8157 - val_specificity: 0.8842 - val_miss_rate: 0.2529 - val_fall_out: 0.1158 - val_mcc: 0.6162
Epoch 5/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3248 - accuracy: 0.8494 - recall: 0.7450 - precision: 0.6940 - AUROC: 0.9104 - AUPRC: 0.7340 - f1_score: 0.7186 - balanced_accuracy: 0.8154 - specificity: 0.8857 - miss_rate: 0.2550 - fall_out: 0.1143 - mcc: 0.6167 - val_loss: 0.3245 - val_accuracy: 0.8492 - val_recall: 0.7702 - val_precision: 0.6848 - val_AUROC: 0.9108 - val_AUPRC: 0.7370 - val_f1_score: 0.7250 - val_balanced_accuracy: 0.8234 - val_specificity: 0.8767 - val_miss_rate: 0.2298 - val_fall_out: 0.1233 - val_mcc: 0.6236
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3204 - accuracy: 0.8524 - recall: 0.7505 - precision: 0.6995 - AUROC: 0.9129 - AUPRC: 0.7414 - f1_score: 0.7241 - balanced_accuracy: 0.8191 - specificity: 0.8878 - miss_rate: 0.2495 - fall_out: 0.1122 - mcc: 0.6242 - val_loss: 0.3259 - val_accuracy: 0.8492 - val_recall: 0.7692 - val_precision: 0.6853 - val_AUROC: 0.9106 - val_AUPRC: 0.7338 - val_f1_score: 0.7248 - val_balanced_accuracy: 0.8232 - val_specificity: 0.8771 - val_miss_rate: 0.2308 - val_fall_out: 0.1229 - val_mcc: 0.6234
Epoch 7/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3179 - accuracy: 0.8518 - recall: 0.7446 - precision: 0.7002 - AUROC: 0.9144 - AUPRC: 0.7481 - f1_score: 0.7217 - balanced_accuracy: 0.8168 - specificity: 0.8891 - miss_rate: 0.2554 - fall_out: 0.1109 - mcc: 0.6214 - val_loss: 0.3239 - val_accuracy: 0.8489 - val_recall: 0.7656 - val_precision: 0.6857 - val_AUROC: 0.9108 - val_AUPRC: 0.7319 - val_f1_score: 0.7234 - val_balanced_accuracy: 0.8217 - val_specificity: 0.8779 - val_miss_rate: 0.2344 - val_fall_out: 0.1221 - val_mcc: 0.6217
Epoch 8/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3153 - accuracy: 0.8541 - recall: 0.7493 - precision: 0.7044 - AUROC: 0.9159 - AUPRC: 0.7525 - f1_score: 0.7262 - balanced_accuracy: 0.8200 - specificity: 0.8906 - miss_rate: 0.2507 - fall_out: 0.1094 - mcc: 0.6275 - val_loss: 0.3264 - val_accuracy: 0.8487 - val_recall: 0.8005 - val_precision: 0.6743 - val_AUROC: 0.9115 - val_AUPRC: 0.7358 - val_f1_score: 0.7320 - val_balanced_accuracy: 0.8330 - val_specificity: 0.8655 - val_miss_rate: 0.1995 - val_fall_out: 0.1345 - val_mcc: 0.6321
Epoch 9/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3140 - accuracy: 0.8555 - recall: 0.7512 - precision: 0.7071 - AUROC: 0.9166 - AUPRC: 0.7523 - f1_score: 0.7285 - balanced_accuracy: 0.8215 - specificity: 0.8917 - miss_rate: 0.2488 - fall_out: 0.1083 - mcc: 0.6307 - val_loss: 0.3237 - val_accuracy: 0.8497 - val_recall: 0.7190 - val_precision: 0.7047 - val_AUROC: 0.9111 - val_AUPRC: 0.7357 - val_f1_score: 0.7118 - val_balanced_accuracy: 0.8071 - val_specificity: 0.8951 - val_miss_rate: 0.2810 - val_fall_out: 0.1049 - val_mcc: 0.6102
Epoch 10/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3107 - accuracy: 0.8549 - recall: 0.7490 - precision: 0.7065 - AUROC: 0.9182 - AUPRC: 0.7610 - f1_score: 0.7271 - balanced_accuracy: 0.8204 - specificity: 0.8917 - miss_rate: 0.2510 - fall_out: 0.1083 - mcc: 0.6289 - val_loss: 0.3254 - val_accuracy: 0.8495 - val_recall: 0.7917 - val_precision: 0.6788 - val_AUROC: 0.9126 - val_AUPRC: 0.7391 - val_f1_score: 0.7309 - val_balanced_accuracy: 0.8307 - val_specificity: 0.8696 - val_miss_rate: 0.2083 - val_fall_out: 0.1304 - val_mcc: 0.6309
Epoch 11/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3085 - accuracy: 0.8577 - recall: 0.7537 - precision: 0.7120 - AUROC: 0.9197 - AUPRC: 0.7631 - f1_score: 0.7323 - balanced_accuracy: 0.8238 - specificity: 0.8939 - miss_rate: 0.2463 - fall_out: 0.1061 - mcc: 0.6360 - val_loss: 0.3194 - val_accuracy: 0.8530 - val_recall: 0.7784 - val_precision: 0.6911 - val_AUROC: 0.9142 - val_AUPRC: 0.7437 - val_f1_score: 0.7321 - val_balanced_accuracy: 0.8287 - val_specificity: 0.8789 - val_miss_rate: 0.2216 - val_fall_out: 0.1211 - val_mcc: 0.6334
Epoch 12/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3067 - accuracy: 0.8579 - recall: 0.7544 - precision: 0.7122 - AUROC: 0.9207 - AUPRC: 0.7646 - f1_score: 0.7327 - balanced_accuracy: 0.8242 - specificity: 0.8939 - miss_rate: 0.2456 - fall_out: 0.1061 - mcc: 0.6365 - val_loss: 0.3200 - val_accuracy: 0.8531 - val_recall: 0.7671 - val_precision: 0.6953 - val_AUROC: 0.9141 - val_AUPRC: 0.7420 - val_f1_score: 0.7294 - val_balanced_accuracy: 0.8251 - val_specificity: 0.8830 - val_miss_rate: 0.2329 - val_fall_out: 0.1170 - val_mcc: 0.6304
Epoch 13/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3052 - accuracy: 0.8601 - recall: 0.7583 - precision: 0.7164 - AUROC: 0.9216 - AUPRC: 0.7707 - f1_score: 0.7367 - balanced_accuracy: 0.8269 - specificity: 0.8956 - miss_rate: 0.2417 - fall_out: 0.1044 - mcc: 0.6421 - val_loss: 0.3217 - val_accuracy: 0.8537 - val_recall: 0.7644 - val_precision: 0.6977 - val_AUROC: 0.9131 - val_AUPRC: 0.7428 - val_f1_score: 0.7295 - val_balanced_accuracy: 0.8246 - val_specificity: 0.8848 - val_miss_rate: 0.2356 - val_fall_out: 0.1152 - val_mcc: 0.6308
-- HOLDOUT 3 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_2\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_14 (Dense) (None, 8) 4504
dropout_7 (Dropout) (None, 8) 0
dense_15 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_2\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 8, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.5}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_16 (Dense) (None, 136) 76160
dropout_8 (Dropout) (None, 136) 0
dense_17 (Dense) (None, 1) 137
=================================================================
Total params: 76,297
Trainable params: 76,297
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3877 - accuracy: 0.8231 - recall: 0.6854 - precision: 0.6489 - AUROC: 0.8771 - AUPRC: 0.6557 - f1_score: 0.6667 - balanced_accuracy: 0.7782 - specificity: 0.8710 - miss_rate: 0.3146 - fall_out: 0.1290 - mcc: 0.5468 - val_loss: 0.3480 - val_accuracy: 0.8372 - val_recall: 0.7287 - val_precision: 0.6696 - val_AUROC: 0.8974 - val_AUPRC: 0.6955 - val_f1_score: 0.6979 - val_balanced_accuracy: 0.8018 - val_specificity: 0.8749 - val_miss_rate: 0.2713 - val_fall_out: 0.1251 - val_mcc: 0.5877
Epoch 2/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3388 - accuracy: 0.8422 - recall: 0.7286 - precision: 0.6820 - AUROC: 0.9025 - AUPRC: 0.7158 - f1_score: 0.7045 - balanced_accuracy: 0.8052 - specificity: 0.8818 - miss_rate: 0.2714 - fall_out: 0.1182 - mcc: 0.5977 - val_loss: 0.3298 - val_accuracy: 0.8470 - val_recall: 0.7200 - val_precision: 0.6971 - val_AUROC: 0.9068 - val_AUPRC: 0.7127 - val_f1_score: 0.7084 - val_balanced_accuracy: 0.8056 - val_specificity: 0.8912 - val_miss_rate: 0.2800 - val_fall_out: 0.1088 - val_mcc: 0.6048
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3278 - accuracy: 0.8478 - recall: 0.7342 - precision: 0.6938 - AUROC: 0.9088 - AUPRC: 0.7320 - f1_score: 0.7134 - balanced_accuracy: 0.8107 - specificity: 0.8873 - miss_rate: 0.2658 - fall_out: 0.1127 - mcc: 0.6103 - val_loss: 0.3310 - val_accuracy: 0.8425 - val_recall: 0.6603 - val_precision: 0.7095 - val_AUROC: 0.9075 - val_AUPRC: 0.7280 - val_f1_score: 0.6840 - val_balanced_accuracy: 0.7831 - val_specificity: 0.9059 - val_miss_rate: 0.3397 - val_fall_out: 0.0941 - val_mcc: 0.5800
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3214 - accuracy: 0.8510 - recall: 0.7375 - precision: 0.7009 - AUROC: 0.9124 - AUPRC: 0.7439 - f1_score: 0.7187 - balanced_accuracy: 0.8140 - specificity: 0.8905 - miss_rate: 0.2625 - fall_out: 0.1095 - mcc: 0.6178 - val_loss: 0.3249 - val_accuracy: 0.8494 - val_recall: 0.7225 - val_precision: 0.7025 - val_AUROC: 0.9100 - val_AUPRC: 0.7301 - val_f1_score: 0.7124 - val_balanced_accuracy: 0.8080 - val_specificity: 0.8935 - val_miss_rate: 0.2775 - val_fall_out: 0.1065 - val_mcc: 0.6105
Epoch 5/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3150 - accuracy: 0.8536 - recall: 0.7398 - precision: 0.7067 - AUROC: 0.9160 - AUPRC: 0.7524 - f1_score: 0.7229 - balanced_accuracy: 0.8165 - specificity: 0.8931 - miss_rate: 0.2602 - fall_out: 0.1069 - mcc: 0.6237 - val_loss: 0.3209 - val_accuracy: 0.8497 - val_recall: 0.7458 - val_precision: 0.6946 - val_AUROC: 0.9121 - val_AUPRC: 0.7351 - val_f1_score: 0.7193 - val_balanced_accuracy: 0.8158 - val_specificity: 0.8859 - val_miss_rate: 0.2542 - val_fall_out: 0.1141 - val_mcc: 0.6176
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3113 - accuracy: 0.8568 - recall: 0.7457 - precision: 0.7127 - AUROC: 0.9182 - AUPRC: 0.7603 - f1_score: 0.7288 - balanced_accuracy: 0.8206 - specificity: 0.8954 - miss_rate: 0.2543 - fall_out: 0.1046 - mcc: 0.6319 - val_loss: 0.3207 - val_accuracy: 0.8496 - val_recall: 0.7039 - val_precision: 0.7108 - val_AUROC: 0.9126 - val_AUPRC: 0.7375 - val_f1_score: 0.7073 - val_balanced_accuracy: 0.8021 - val_specificity: 0.9003 - val_miss_rate: 0.2961 - val_fall_out: 0.0997 - val_mcc: 0.6062
Epoch 7/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3072 - accuracy: 0.8593 - recall: 0.7496 - precision: 0.7177 - AUROC: 0.9205 - AUPRC: 0.7662 - f1_score: 0.7333 - balanced_accuracy: 0.8235 - specificity: 0.8974 - miss_rate: 0.2504 - fall_out: 0.1026 - mcc: 0.6380 - val_loss: 0.3299 - val_accuracy: 0.8481 - val_recall: 0.7568 - val_precision: 0.6867 - val_AUROC: 0.9082 - val_AUPRC: 0.7325 - val_f1_score: 0.7200 - val_balanced_accuracy: 0.8183 - val_specificity: 0.8798 - val_miss_rate: 0.2432 - val_fall_out: 0.1202 - val_mcc: 0.6175
Epoch 8/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3033 - accuracy: 0.8612 - recall: 0.7538 - precision: 0.7210 - AUROC: 0.9226 - AUPRC: 0.7733 - f1_score: 0.7370 - balanced_accuracy: 0.8262 - specificity: 0.8985 - miss_rate: 0.2462 - fall_out: 0.1015 - mcc: 0.6431 - val_loss: 0.3283 - val_accuracy: 0.8483 - val_recall: 0.8084 - val_precision: 0.6711 - val_AUROC: 0.9130 - val_AUPRC: 0.7387 - val_f1_score: 0.7334 - val_balanced_accuracy: 0.8353 - val_specificity: 0.8621 - val_miss_rate: 0.1916 - val_fall_out: 0.1379 - val_mcc: 0.6340
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 8 / 100
Confirmed: 399
Tentative: 160
Rejected: 0
Iteration: 9 / 100
Confirmed: 399
Tentative: 160
Rejected: 0
Iteration: 10 / 100
Confirmed: 399
Tentative: 160
Rejected: 0
Iteration: 11 / 100
Confirmed: 399
Tentative: 160
Rejected: 0
Iteration: 12 / 100
Confirmed: 428
Tentative: 127
Rejected: 4
Iteration: 13 / 100
Confirmed: 428
Tentative: 127
Rejected: 4
Iteration: 14 / 100
Confirmed: 428
Tentative: 127
Rejected: 4
Iteration: 15 / 100
Confirmed: 428
Tentative: 127
Rejected: 4
Iteration: 16 / 100
Confirmed: 443
Tentative: 112
Rejected: 4
Iteration: 17 / 100
Confirmed: 443
Tentative: 112
Rejected: 4
Iteration: 18 / 100
Confirmed: 443
Tentative: 111
Rejected: 5
Iteration: 19 / 100
Confirmed: 455
Tentative: 99
Rejected: 5
Iteration: 20 / 100
Confirmed: 455
Tentative: 99
Rejected: 5
Iteration: 21 / 100
Confirmed: 455
Tentative: 99
Rejected: 5
Iteration: 22 / 100
Confirmed: 458
Tentative: 96
Rejected: 5
Iteration: 23 / 100
Confirmed: 458
Tentative: 96
Rejected: 5
Iteration: 24 / 100
Confirmed: 458
Tentative: 96
Rejected: 5
Iteration: 25 / 100
Confirmed: 458
Tentative: 96
Rejected: 5
Iteration: 26 / 100
Confirmed: 460
Tentative: 91
Rejected: 8
Iteration: 27 / 100
Confirmed: 460
Tentative: 91
Rejected: 8
Iteration: 28 / 100
Confirmed: 460
Tentative: 91
Rejected: 8
Iteration: 29 / 100
Confirmed: 463
Tentative: 88
Rejected: 8
Iteration: 30 / 100
Confirmed: 463
Tentative: 86
Rejected: 10
Iteration: 31 / 100
Confirmed: 463
Tentative: 86
Rejected: 10
Iteration: 32 / 100
Confirmed: 463
Tentative: 86
Rejected: 10
Iteration: 33 / 100
Confirmed: 463
Tentative: 86
Rejected: 10
Iteration: 34 / 100
Confirmed: 464
Tentative: 85
Rejected: 10
Iteration: 35 / 100
Confirmed: 464
Tentative: 85
Rejected: 10
Iteration: 36 / 100
Confirmed: 464
Tentative: 85
Rejected: 10
Iteration: 37 / 100
Confirmed: 467
Tentative: 82
Rejected: 10
Iteration: 38 / 100
Confirmed: 467
Tentative: 82
Rejected: 10
Iteration: 39 / 100
Confirmed: 467
Tentative: 82
Rejected: 10
Iteration: 40 / 100
Confirmed: 469
Tentative: 80
Rejected: 10
Iteration: 41 / 100
Confirmed: 469
Tentative: 79
Rejected: 11
Iteration: 42 / 100
Confirmed: 469
Tentative: 79
Rejected: 11
Iteration: 43 / 100
Confirmed: 470
Tentative: 78
Rejected: 11
Iteration: 44 / 100
Confirmed: 470
Tentative: 78
Rejected: 11
Iteration: 45 / 100
Confirmed: 470
Tentative: 78
Rejected: 11
Iteration: 46 / 100
Confirmed: 470
Tentative: 78
Rejected: 11
Iteration: 47 / 100
Confirmed: 470
Tentative: 78
Rejected: 11
Iteration: 48 / 100
Confirmed: 470
Tentative: 78
Rejected: 11
Iteration: 49 / 100
Confirmed: 471
Tentative: 76
Rejected: 12
Iteration: 50 / 100
Confirmed: 471
Tentative: 76
Rejected: 12
Iteration: 51 / 100
Confirmed: 475
Tentative: 72
Rejected: 12
Iteration: 52 / 100
Confirmed: 475
Tentative: 71
Rejected: 13
Iteration: 53 / 100
Confirmed: 475
Tentative: 71
Rejected: 13
Iteration: 54 / 100
Confirmed: 478
Tentative: 65
Rejected: 16
Iteration: 55 / 100
Confirmed: 478
Tentative: 65
Rejected: 16
Iteration: 56 / 100
Confirmed: 478
Tentative: 65
Rejected: 16
Iteration: 57 / 100
Confirmed: 480
Tentative: 63
Rejected: 16
Iteration: 58 / 100
Confirmed: 480
Tentative: 63
Rejected: 16
Iteration: 59 / 100
Confirmed: 481
Tentative: 62
Rejected: 16
Iteration: 60 / 100
Confirmed: 481
Tentative: 62
Rejected: 16
Iteration: 61 / 100
Confirmed: 481
Tentative: 62
Rejected: 16
Iteration: 62 / 100
Confirmed: 482
Tentative: 60
Rejected: 17
Iteration: 63 / 100
Confirmed: 482
Tentative: 60
Rejected: 17
Iteration: 64 / 100
Confirmed: 482
Tentative: 60
Rejected: 17
Iteration: 65 / 100
Confirmed: 483
Tentative: 59
Rejected: 17
Iteration: 66 / 100
Confirmed: 483
Tentative: 59
Rejected: 17
Iteration: 67 / 100
Confirmed: 484
Tentative: 58
Rejected: 17
Iteration: 68 / 100
Confirmed: 484
Tentative: 58
Rejected: 17
Iteration: 69 / 100
Confirmed: 484
Tentative: 58
Rejected: 17
Iteration: 70 / 100
Confirmed: 485
Tentative: 56
Rejected: 18
Iteration: 71 / 100
Confirmed: 485
Tentative: 56
Rejected: 18
Iteration: 72 / 100
Confirmed: 486
Tentative: 55
Rejected: 18
Iteration: 73 / 100
Confirmed: 486
Tentative: 55
Rejected: 18
Iteration: 74 / 100
Confirmed: 486
Tentative: 55
Rejected: 18
Iteration: 75 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 76 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 77 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 78 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 79 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 80 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 81 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 82 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 83 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 84 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 85 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 86 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 87 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 88 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 89 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 90 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 91 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 92 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 93 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 94 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 95 / 100
Confirmed: 488
Tentative: 53
Rejected: 18
Iteration: 96 / 100
Confirmed: 488
Tentative: 48
Rejected: 23
Iteration: 97 / 100
Confirmed: 488
Tentative: 48
Rejected: 23
Iteration: 98 / 100
Confirmed: 488
Tentative: 48
Rejected: 23
Iteration: 99 / 100
Confirmed: 488
Tentative: 48
Rejected: 23
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 488
Tentative: 24
Rejected: 23
-- 488 Kept (correlated) features [Boruta]
-- 71 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_18 (Dense) (None, 136) 66504
dropout_9 (Dropout) (None, 136) 0
dense_19 (Dense) (None, 1) 137
=================================================================
Total params: 66,641
Trainable params: 66,641
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.4011 - accuracy: 0.8195 - recall: 0.6763 - precision: 0.6430 - AUROC: 0.8718 - AUPRC: 0.6503 - f1_score: 0.6592 - balanced_accuracy: 0.7728 - specificity: 0.8693 - miss_rate: 0.3237 - fall_out: 0.1307 - mcc: 0.5369 - val_loss: 0.3395 - val_accuracy: 0.8424 - val_recall: 0.7118 - val_precision: 0.6884 - val_AUROC: 0.9014 - val_AUPRC: 0.7031 - val_f1_score: 0.6999 - val_balanced_accuracy: 0.7999 - val_specificity: 0.8879 - val_miss_rate: 0.2882 - val_fall_out: 0.1121 - val_mcc: 0.5933
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3429 - accuracy: 0.8424 - recall: 0.7290 - precision: 0.6822 - AUROC: 0.9006 - AUPRC: 0.7101 - f1_score: 0.7048 - balanced_accuracy: 0.8054 - specificity: 0.8818 - miss_rate: 0.2710 - fall_out: 0.1182 - mcc: 0.5981 - val_loss: 0.3320 - val_accuracy: 0.8442 - val_recall: 0.7369 - val_precision: 0.6839 - val_AUROC: 0.9059 - val_AUPRC: 0.7166 - val_f1_score: 0.7094 - val_balanced_accuracy: 0.8092 - val_specificity: 0.8815 - val_miss_rate: 0.2631 - val_fall_out: 0.1185 - val_mcc: 0.6039
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3305 - accuracy: 0.8473 - recall: 0.7346 - precision: 0.6924 - AUROC: 0.9073 - AUPRC: 0.7261 - f1_score: 0.7129 - balanced_accuracy: 0.8106 - specificity: 0.8865 - miss_rate: 0.2654 - fall_out: 0.1135 - mcc: 0.6095 - val_loss: 0.3296 - val_accuracy: 0.8465 - val_recall: 0.7551 - val_precision: 0.6834 - val_AUROC: 0.9066 - val_AUPRC: 0.7206 - val_f1_score: 0.7175 - val_balanced_accuracy: 0.8167 - val_specificity: 0.8783 - val_miss_rate: 0.2449 - val_fall_out: 0.1217 - val_mcc: 0.6139
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3237 - accuracy: 0.8501 - recall: 0.7388 - precision: 0.6981 - AUROC: 0.9112 - AUPRC: 0.7405 - f1_score: 0.7178 - balanced_accuracy: 0.8138 - specificity: 0.8888 - miss_rate: 0.2612 - fall_out: 0.1112 - mcc: 0.6163 - val_loss: 0.3489 - val_accuracy: 0.8327 - val_recall: 0.6199 - val_precision: 0.6980 - val_AUROC: 0.8954 - val_AUPRC: 0.6939 - val_f1_score: 0.6567 - val_balanced_accuracy: 0.7633 - val_specificity: 0.9067 - val_miss_rate: 0.3801 - val_fall_out: 0.0933 - val_mcc: 0.5482
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3194 - accuracy: 0.8518 - recall: 0.7404 - precision: 0.7019 - AUROC: 0.9136 - AUPRC: 0.7459 - f1_score: 0.7206 - balanced_accuracy: 0.8155 - specificity: 0.8906 - miss_rate: 0.2596 - fall_out: 0.1094 - mcc: 0.6203 - val_loss: 0.3231 - val_accuracy: 0.8479 - val_recall: 0.7029 - val_precision: 0.7065 - val_AUROC: 0.9108 - val_AUPRC: 0.7330 - val_f1_score: 0.7047 - val_balanced_accuracy: 0.8007 - val_specificity: 0.8984 - val_miss_rate: 0.2971 - val_fall_out: 0.1016 - val_mcc: 0.6023
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3150 - accuracy: 0.8543 - recall: 0.7440 - precision: 0.7069 - AUROC: 0.9161 - AUPRC: 0.7562 - f1_score: 0.7250 - balanced_accuracy: 0.8183 - specificity: 0.8927 - miss_rate: 0.2560 - fall_out: 0.1073 - mcc: 0.6263 - val_loss: 0.3211 - val_accuracy: 0.8494 - val_recall: 0.7066 - val_precision: 0.7089 - val_AUROC: 0.9122 - val_AUPRC: 0.7349 - val_f1_score: 0.7078 - val_balanced_accuracy: 0.8028 - val_specificity: 0.8991 - val_miss_rate: 0.2934 - val_fall_out: 0.1009 - val_mcc: 0.6063
Epoch 7/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3088 - accuracy: 0.8583 - recall: 0.7506 - precision: 0.7146 - AUROC: 0.9196 - AUPRC: 0.7649 - f1_score: 0.7322 - balanced_accuracy: 0.8232 - specificity: 0.8957 - miss_rate: 0.2494 - fall_out: 0.1043 - mcc: 0.6363 - val_loss: 0.3312 - val_accuracy: 0.8478 - val_recall: 0.8084 - val_precision: 0.6700 - val_AUROC: 0.9107 - val_AUPRC: 0.7293 - val_f1_score: 0.7328 - val_balanced_accuracy: 0.8349 - val_specificity: 0.8615 - val_miss_rate: 0.1916 - val_fall_out: 0.1385 - val_mcc: 0.6330
Epoch 8/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3064 - accuracy: 0.8586 - recall: 0.7499 - precision: 0.7159 - AUROC: 0.9207 - AUPRC: 0.7699 - f1_score: 0.7325 - balanced_accuracy: 0.8232 - specificity: 0.8965 - miss_rate: 0.2501 - fall_out: 0.1035 - mcc: 0.6368 - val_loss: 0.3219 - val_accuracy: 0.8507 - val_recall: 0.7836 - val_precision: 0.6840 - val_AUROC: 0.9129 - val_AUPRC: 0.7378 - val_f1_score: 0.7304 - val_balanced_accuracy: 0.8288 - val_specificity: 0.8740 - val_miss_rate: 0.2164 - val_fall_out: 0.1260 - val_mcc: 0.6306
-- HOLDOUT 4 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_3\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_20 (Dense) (None, 8) 4504
dropout_10 (Dropout) (None, 8) 0
dense_21 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_3\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 40, 'dropout_0': 0.5, 'units_1': 72, 'dropout_1': 0.5, 'units_2': 136, 'dropout_2': 0.5}
-- 3 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'NBN', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'NBN', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_22 (Dense) (None, 40) 22360
dropout_11 (Dropout) (None, 40) 0
dense_23 (Dense) (None, 1) 41
=================================================================
Total params: 22,401
Trainable params: 22,401
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.4301 - accuracy: 0.7982 - recall: 0.6919 - precision: 0.5936 - AUROC: 0.8522 - AUPRC: 0.6021 - f1_score: 0.6390 - balanced_accuracy: 0.7635 - specificity: 0.8352 - miss_rate: 0.3081 - fall_out: 0.1648 - mcc: 0.5029 - val_loss: 0.3404 - val_accuracy: 0.8393 - val_recall: 0.7400 - val_precision: 0.6711 - val_AUROC: 0.9017 - val_AUPRC: 0.7059 - val_f1_score: 0.7039 - val_balanced_accuracy: 0.8069 - val_specificity: 0.8738 - val_miss_rate: 0.2600 - val_fall_out: 0.1262 - val_mcc: 0.5953
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3588 - accuracy: 0.8329 - recall: 0.7264 - precision: 0.6602 - AUROC: 0.8908 - AUPRC: 0.6861 - f1_score: 0.6917 - balanced_accuracy: 0.7982 - specificity: 0.8699 - miss_rate: 0.2736 - fall_out: 0.1301 - mcc: 0.5787 - val_loss: 0.3315 - val_accuracy: 0.8415 - val_recall: 0.7768 - val_precision: 0.6652 - val_AUROC: 0.9056 - val_AUPRC: 0.7138 - val_f1_score: 0.7167 - val_balanced_accuracy: 0.8204 - val_specificity: 0.8640 - val_miss_rate: 0.2232 - val_fall_out: 0.1360 - val_mcc: 0.6111
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3483 - accuracy: 0.8370 - recall: 0.7284 - precision: 0.6693 - AUROC: 0.8969 - AUPRC: 0.7015 - f1_score: 0.6976 - balanced_accuracy: 0.8016 - specificity: 0.8748 - miss_rate: 0.2716 - fall_out: 0.1252 - mcc: 0.5873 - val_loss: 0.3292 - val_accuracy: 0.8439 - val_recall: 0.6961 - val_precision: 0.6983 - val_AUROC: 0.9080 - val_AUPRC: 0.7206 - val_f1_score: 0.6972 - val_balanced_accuracy: 0.7957 - val_specificity: 0.8953 - val_miss_rate: 0.3039 - val_fall_out: 0.1047 - val_mcc: 0.5921
Epoch 4/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3422 - accuracy: 0.8406 - recall: 0.7252 - precision: 0.6790 - AUROC: 0.9007 - AUPRC: 0.7131 - f1_score: 0.7014 - balanced_accuracy: 0.8030 - specificity: 0.8807 - miss_rate: 0.2748 - fall_out: 0.1193 - mcc: 0.5934 - val_loss: 0.3257 - val_accuracy: 0.8471 - val_recall: 0.7675 - val_precision: 0.6808 - val_AUROC: 0.9093 - val_AUPRC: 0.7241 - val_f1_score: 0.7215 - val_balanced_accuracy: 0.8211 - val_specificity: 0.8748 - val_miss_rate: 0.2325 - val_fall_out: 0.1252 - val_mcc: 0.6188
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3398 - accuracy: 0.8426 - recall: 0.7236 - precision: 0.6845 - AUROC: 0.9022 - AUPRC: 0.7135 - f1_score: 0.7035 - balanced_accuracy: 0.8038 - specificity: 0.8839 - miss_rate: 0.2764 - fall_out: 0.1161 - mcc: 0.5969 - val_loss: 0.3226 - val_accuracy: 0.8475 - val_recall: 0.7289 - val_precision: 0.6951 - val_AUROC: 0.9110 - val_AUPRC: 0.7296 - val_f1_score: 0.7116 - val_balanced_accuracy: 0.8088 - val_specificity: 0.8887 - val_miss_rate: 0.2711 - val_fall_out: 0.1113 - val_mcc: 0.6083
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3376 - accuracy: 0.8440 - recall: 0.7148 - precision: 0.6913 - AUROC: 0.9037 - AUPRC: 0.7195 - f1_score: 0.7028 - balanced_accuracy: 0.8019 - specificity: 0.8889 - miss_rate: 0.2852 - fall_out: 0.1111 - mcc: 0.5973 - val_loss: 0.3225 - val_accuracy: 0.8460 - val_recall: 0.6919 - val_precision: 0.7058 - val_AUROC: 0.9119 - val_AUPRC: 0.7337 - val_f1_score: 0.6988 - val_balanced_accuracy: 0.7958 - val_specificity: 0.8997 - val_miss_rate: 0.3081 - val_fall_out: 0.1003 - val_mcc: 0.5954
Epoch 7/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3338 - accuracy: 0.8465 - recall: 0.7171 - precision: 0.6969 - AUROC: 0.9062 - AUPRC: 0.7251 - f1_score: 0.7069 - balanced_accuracy: 0.8043 - specificity: 0.8915 - miss_rate: 0.2829 - fall_out: 0.1085 - mcc: 0.6030 - val_loss: 0.3194 - val_accuracy: 0.8477 - val_recall: 0.7118 - val_precision: 0.7022 - val_AUROC: 0.9130 - val_AUPRC: 0.7378 - val_f1_score: 0.7070 - val_balanced_accuracy: 0.8034 - val_specificity: 0.8949 - val_miss_rate: 0.2882 - val_fall_out: 0.1051 - val_mcc: 0.6041
Epoch 8/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3306 - accuracy: 0.8472 - recall: 0.7117 - precision: 0.7010 - AUROC: 0.9078 - AUPRC: 0.7309 - f1_score: 0.7063 - balanced_accuracy: 0.8030 - specificity: 0.8944 - miss_rate: 0.2883 - fall_out: 0.1056 - mcc: 0.6031 - val_loss: 0.3182 - val_accuracy: 0.8489 - val_recall: 0.7609 - val_precision: 0.6872 - val_AUROC: 0.9132 - val_AUPRC: 0.7360 - val_f1_score: 0.7222 - val_balanced_accuracy: 0.8202 - val_specificity: 0.8795 - val_miss_rate: 0.2391 - val_fall_out: 0.1205 - val_mcc: 0.6203
Epoch 9/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3292 - accuracy: 0.8483 - recall: 0.7108 - precision: 0.7043 - AUROC: 0.9087 - AUPRC: 0.7330 - f1_score: 0.7076 - balanced_accuracy: 0.8035 - specificity: 0.8962 - miss_rate: 0.2892 - fall_out: 0.1038 - mcc: 0.6052 - val_loss: 0.3172 - val_accuracy: 0.8495 - val_recall: 0.7665 - val_precision: 0.6868 - val_AUROC: 0.9139 - val_AUPRC: 0.7381 - val_f1_score: 0.7245 - val_balanced_accuracy: 0.8224 - val_specificity: 0.8783 - val_miss_rate: 0.2335 - val_fall_out: 0.1217 - val_mcc: 0.6231
Epoch 10/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3260 - accuracy: 0.8482 - recall: 0.7117 - precision: 0.7035 - AUROC: 0.9102 - AUPRC: 0.7362 - f1_score: 0.7076 - balanced_accuracy: 0.8037 - specificity: 0.8957 - miss_rate: 0.2883 - fall_out: 0.1043 - mcc: 0.6051 - val_loss: 0.3171 - val_accuracy: 0.8512 - val_recall: 0.7774 - val_precision: 0.6872 - val_AUROC: 0.9145 - val_AUPRC: 0.7396 - val_f1_score: 0.7295 - val_balanced_accuracy: 0.8271 - val_specificity: 0.8769 - val_miss_rate: 0.2226 - val_fall_out: 0.1231 - val_mcc: 0.6297
Epoch 11/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3256 - accuracy: 0.8489 - recall: 0.7117 - precision: 0.7056 - AUROC: 0.9106 - AUPRC: 0.7410 - f1_score: 0.7086 - balanced_accuracy: 0.8042 - specificity: 0.8967 - miss_rate: 0.2883 - fall_out: 0.1033 - mcc: 0.6067 - val_loss: 0.3168 - val_accuracy: 0.8494 - val_recall: 0.7146 - val_precision: 0.7057 - val_AUROC: 0.9141 - val_AUPRC: 0.7428 - val_f1_score: 0.7101 - val_balanced_accuracy: 0.8054 - val_specificity: 0.8963 - val_miss_rate: 0.2854 - val_fall_out: 0.1037 - val_mcc: 0.6084
Epoch 12/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3250 - accuracy: 0.8493 - recall: 0.7061 - precision: 0.7090 - AUROC: 0.9109 - AUPRC: 0.7413 - f1_score: 0.7076 - balanced_accuracy: 0.8026 - specificity: 0.8992 - miss_rate: 0.2939 - fall_out: 0.1008 - mcc: 0.6061 - val_loss: 0.3164 - val_accuracy: 0.8510 - val_recall: 0.7458 - val_precision: 0.6979 - val_AUROC: 0.9146 - val_AUPRC: 0.7423 - val_f1_score: 0.7210 - val_balanced_accuracy: 0.8167 - val_specificity: 0.8877 - val_miss_rate: 0.2542 - val_fall_out: 0.1123 - val_mcc: 0.6202
Epoch 13/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3236 - accuracy: 0.8491 - recall: 0.7087 - precision: 0.7074 - AUROC: 0.9117 - AUPRC: 0.7416 - f1_score: 0.7080 - balanced_accuracy: 0.8033 - specificity: 0.8980 - miss_rate: 0.2913 - fall_out: 0.1020 - mcc: 0.6063 - val_loss: 0.3199 - val_accuracy: 0.8516 - val_recall: 0.7543 - val_precision: 0.6963 - val_AUROC: 0.9138 - val_AUPRC: 0.7420 - val_f1_score: 0.7241 - val_balanced_accuracy: 0.8199 - val_specificity: 0.8855 - val_miss_rate: 0.2457 - val_fall_out: 0.1145 - val_mcc: 0.6238
Epoch 14/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3211 - accuracy: 0.8518 - recall: 0.7135 - precision: 0.7127 - AUROC: 0.9133 - AUPRC: 0.7469 - f1_score: 0.7131 - balanced_accuracy: 0.8067 - specificity: 0.8999 - miss_rate: 0.2865 - fall_out: 0.1001 - mcc: 0.6132 - val_loss: 0.3141 - val_accuracy: 0.8509 - val_recall: 0.7231 - val_precision: 0.7063 - val_AUROC: 0.9159 - val_AUPRC: 0.7441 - val_f1_score: 0.7146 - val_balanced_accuracy: 0.8092 - val_specificity: 0.8953 - val_miss_rate: 0.2769 - val_fall_out: 0.1047 - val_mcc: 0.6137
Epoch 15/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3208 - accuracy: 0.8515 - recall: 0.7127 - precision: 0.7121 - AUROC: 0.9135 - AUPRC: 0.7481 - f1_score: 0.7124 - balanced_accuracy: 0.8062 - specificity: 0.8997 - miss_rate: 0.2873 - fall_out: 0.1003 - mcc: 0.6123 - val_loss: 0.3134 - val_accuracy: 0.8511 - val_recall: 0.7365 - val_precision: 0.7015 - val_AUROC: 0.9160 - val_AUPRC: 0.7436 - val_f1_score: 0.7186 - val_balanced_accuracy: 0.8137 - val_specificity: 0.8910 - val_miss_rate: 0.2635 - val_fall_out: 0.1090 - val_mcc: 0.6177
Epoch 16/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3190 - accuracy: 0.8515 - recall: 0.7082 - precision: 0.7142 - AUROC: 0.9144 - AUPRC: 0.7509 - f1_score: 0.7112 - balanced_accuracy: 0.8048 - specificity: 0.9014 - miss_rate: 0.2918 - fall_out: 0.0986 - mcc: 0.6113 - val_loss: 0.3126 - val_accuracy: 0.8520 - val_recall: 0.7407 - val_precision: 0.7022 - val_AUROC: 0.9168 - val_AUPRC: 0.7480 - val_f1_score: 0.7210 - val_balanced_accuracy: 0.8157 - val_specificity: 0.8907 - val_miss_rate: 0.2593 - val_fall_out: 0.1093 - val_mcc: 0.6207
Epoch 17/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3183 - accuracy: 0.8530 - recall: 0.7102 - precision: 0.7174 - AUROC: 0.9150 - AUPRC: 0.7535 - f1_score: 0.7138 - balanced_accuracy: 0.8064 - specificity: 0.9026 - miss_rate: 0.2898 - fall_out: 0.0974 - mcc: 0.6149 - val_loss: 0.3122 - val_accuracy: 0.8518 - val_recall: 0.7345 - val_precision: 0.7042 - val_AUROC: 0.9170 - val_AUPRC: 0.7503 - val_f1_score: 0.7191 - val_balanced_accuracy: 0.8136 - val_specificity: 0.8926 - val_miss_rate: 0.2655 - val_fall_out: 0.1074 - val_mcc: 0.6188
Epoch 18/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3171 - accuracy: 0.8519 - recall: 0.7065 - precision: 0.7159 - AUROC: 0.9154 - AUPRC: 0.7552 - f1_score: 0.7112 - balanced_accuracy: 0.8045 - specificity: 0.9025 - miss_rate: 0.2935 - fall_out: 0.0975 - mcc: 0.6116 - val_loss: 0.3141 - val_accuracy: 0.8522 - val_recall: 0.7060 - val_precision: 0.7172 - val_AUROC: 0.9166 - val_AUPRC: 0.7493 - val_f1_score: 0.7115 - val_balanced_accuracy: 0.8046 - val_specificity: 0.9031 - val_miss_rate: 0.2940 - val_fall_out: 0.0969 - val_mcc: 0.6123
Epoch 19/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3146 - accuracy: 0.8541 - recall: 0.7128 - precision: 0.7195 - AUROC: 0.9167 - AUPRC: 0.7603 - f1_score: 0.7161 - balanced_accuracy: 0.8081 - specificity: 0.9033 - miss_rate: 0.2872 - fall_out: 0.0967 - mcc: 0.6180 - val_loss: 0.3142 - val_accuracy: 0.8514 - val_recall: 0.7080 - val_precision: 0.7139 - val_AUROC: 0.9166 - val_AUPRC: 0.7492 - val_f1_score: 0.7109 - val_balanced_accuracy: 0.8046 - val_specificity: 0.9013 - val_miss_rate: 0.2920 - val_fall_out: 0.0987 - val_mcc: 0.6109
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 8 / 100
Confirmed: 408
Tentative: 150
Rejected: 0
Iteration: 9 / 100
Confirmed: 408
Tentative: 150
Rejected: 0
Iteration: 10 / 100
Confirmed: 408
Tentative: 150
Rejected: 0
Iteration: 11 / 100
Confirmed: 408
Tentative: 150
Rejected: 0
Iteration: 12 / 100
Confirmed: 429
Tentative: 125
Rejected: 4
Iteration: 13 / 100
Confirmed: 429
Tentative: 125
Rejected: 4
Iteration: 14 / 100
Confirmed: 429
Tentative: 125
Rejected: 4
Iteration: 15 / 100
Confirmed: 429
Tentative: 125
Rejected: 4
Iteration: 16 / 100
Confirmed: 437
Tentative: 117
Rejected: 4
Iteration: 17 / 100
Confirmed: 437
Tentative: 117
Rejected: 4
Iteration: 18 / 100
Confirmed: 437
Tentative: 116
Rejected: 5
Iteration: 19 / 100
Confirmed: 447
Tentative: 106
Rejected: 5
Iteration: 20 / 100
Confirmed: 447
Tentative: 106
Rejected: 5
Iteration: 21 / 100
Confirmed: 447
Tentative: 106
Rejected: 5
Iteration: 22 / 100
Confirmed: 454
Tentative: 98
Rejected: 6
Iteration: 23 / 100
Confirmed: 454
Tentative: 98
Rejected: 6
Iteration: 24 / 100
Confirmed: 454
Tentative: 96
Rejected: 8
Iteration: 25 / 100
Confirmed: 454
Tentative: 96
Rejected: 8
Iteration: 26 / 100
Confirmed: 463
Tentative: 87
Rejected: 8
Iteration: 27 / 100
Confirmed: 463
Tentative: 87
Rejected: 8
Iteration: 28 / 100
Confirmed: 463
Tentative: 87
Rejected: 8
Iteration: 29 / 100
Confirmed: 466
Tentative: 84
Rejected: 8
Iteration: 30 / 100
Confirmed: 466
Tentative: 84
Rejected: 8
Iteration: 31 / 100
Confirmed: 466
Tentative: 84
Rejected: 8
Iteration: 32 / 100
Confirmed: 468
Tentative: 82
Rejected: 8
Iteration: 33 / 100
Confirmed: 468
Tentative: 82
Rejected: 8
Iteration: 34 / 100
Confirmed: 468
Tentative: 82
Rejected: 8
Iteration: 35 / 100
Confirmed: 468
Tentative: 82
Rejected: 8
Iteration: 36 / 100
Confirmed: 468
Tentative: 82
Rejected: 8
Iteration: 37 / 100
Confirmed: 471
Tentative: 79
Rejected: 8
Iteration: 38 / 100
Confirmed: 471
Tentative: 76
Rejected: 11
Iteration: 39 / 100
Confirmed: 471
Tentative: 76
Rejected: 11
Iteration: 40 / 100
Confirmed: 474
Tentative: 73
Rejected: 11
Iteration: 41 / 100
Confirmed: 474
Tentative: 73
Rejected: 11
Iteration: 42 / 100
Confirmed: 474
Tentative: 73
Rejected: 11
Iteration: 43 / 100
Confirmed: 476
Tentative: 71
Rejected: 11
Iteration: 44 / 100
Confirmed: 476
Tentative: 71
Rejected: 11
Iteration: 45 / 100
Confirmed: 476
Tentative: 71
Rejected: 11
Iteration: 46 / 100
Confirmed: 476
Tentative: 70
Rejected: 12
Iteration: 47 / 100
Confirmed: 476
Tentative: 70
Rejected: 12
Iteration: 48 / 100
Confirmed: 476
Tentative: 70
Rejected: 12
Iteration: 49 / 100
Confirmed: 478
Tentative: 68
Rejected: 12
Iteration: 50 / 100
Confirmed: 478
Tentative: 68
Rejected: 12
Iteration: 51 / 100
Confirmed: 478
Tentative: 68
Rejected: 12
Iteration: 52 / 100
Confirmed: 478
Tentative: 68
Rejected: 12
Iteration: 53 / 100
Confirmed: 478
Tentative: 68
Rejected: 12
Iteration: 54 / 100
Confirmed: 479
Tentative: 66
Rejected: 13
Iteration: 55 / 100
Confirmed: 479
Tentative: 66
Rejected: 13
Iteration: 56 / 100
Confirmed: 479
Tentative: 66
Rejected: 13
Iteration: 57 / 100
Confirmed: 479
Tentative: 66
Rejected: 13
Iteration: 58 / 100
Confirmed: 479
Tentative: 66
Rejected: 13
Iteration: 59 / 100
Confirmed: 480
Tentative: 65
Rejected: 13
Iteration: 60 / 100
Confirmed: 480
Tentative: 62
Rejected: 16
Iteration: 61 / 100
Confirmed: 480
Tentative: 62
Rejected: 16
Iteration: 62 / 100
Confirmed: 481
Tentative: 61
Rejected: 16
Iteration: 63 / 100
Confirmed: 481
Tentative: 61
Rejected: 16
Iteration: 64 / 100
Confirmed: 481
Tentative: 61
Rejected: 16
Iteration: 65 / 100
Confirmed: 482
Tentative: 60
Rejected: 16
Iteration: 66 / 100
Confirmed: 482
Tentative: 60
Rejected: 16
Iteration: 67 / 100
Confirmed: 483
Tentative: 59
Rejected: 16
Iteration: 68 / 100
Confirmed: 483
Tentative: 58
Rejected: 17
Iteration: 69 / 100
Confirmed: 483
Tentative: 58
Rejected: 17
Iteration: 70 / 100
Confirmed: 485
Tentative: 56
Rejected: 17
Iteration: 71 / 100
Confirmed: 485
Tentative: 56
Rejected: 17
Iteration: 72 / 100
Confirmed: 485
Tentative: 53
Rejected: 20
Iteration: 73 / 100
Confirmed: 485
Tentative: 53
Rejected: 20
Iteration: 74 / 100
Confirmed: 485
Tentative: 53
Rejected: 20
Iteration: 75 / 100
Confirmed: 486
Tentative: 52
Rejected: 20
Iteration: 76 / 100
Confirmed: 486
Tentative: 52
Rejected: 20
Iteration: 77 / 100
Confirmed: 487
Tentative: 51
Rejected: 20
Iteration: 78 / 100
Confirmed: 487
Tentative: 51
Rejected: 20
Iteration: 79 / 100
Confirmed: 487
Tentative: 51
Rejected: 20
Iteration: 80 / 100
Confirmed: 488
Tentative: 50
Rejected: 20
Iteration: 81 / 100
Confirmed: 488
Tentative: 50
Rejected: 20
Iteration: 82 / 100
Confirmed: 488
Tentative: 50
Rejected: 20
Iteration: 83 / 100
Confirmed: 489
Tentative: 49
Rejected: 20
Iteration: 84 / 100
Confirmed: 489
Tentative: 49
Rejected: 20
Iteration: 85 / 100
Confirmed: 490
Tentative: 48
Rejected: 20
Iteration: 86 / 100
Confirmed: 490
Tentative: 48
Rejected: 20
Iteration: 87 / 100
Confirmed: 490
Tentative: 48
Rejected: 20
Iteration: 88 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 89 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 90 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 91 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 92 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 93 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 94 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 95 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 96 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 97 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 98 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
Iteration: 99 / 100
Confirmed: 490
Tentative: 47
Rejected: 21
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 490
Tentative: 18
Rejected: 21
-- 490 Kept (correlated) features [Boruta]
-- 68 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_24 (Dense) (None, 40) 19640
dropout_12 (Dropout) (None, 40) 0
dense_25 (Dense) (None, 1) 41
=================================================================
Total params: 19,681
Trainable params: 19,681
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 8s 21ms/step - loss: 0.4273 - accuracy: 0.8053 - recall: 0.6874 - precision: 0.6087 - AUROC: 0.8543 - AUPRC: 0.6094 - f1_score: 0.6457 - balanced_accuracy: 0.7668 - specificity: 0.8463 - miss_rate: 0.3126 - fall_out: 0.1537 - mcc: 0.5139 - val_loss: 0.3409 - val_accuracy: 0.8404 - val_recall: 0.7095 - val_precision: 0.6841 - val_AUROC: 0.9021 - val_AUPRC: 0.7048 - val_f1_score: 0.6966 - val_balanced_accuracy: 0.7977 - val_specificity: 0.8860 - val_miss_rate: 0.2905 - val_fall_out: 0.1140 - val_mcc: 0.5885
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3594 - accuracy: 0.8339 - recall: 0.7156 - precision: 0.6657 - AUROC: 0.8907 - AUPRC: 0.6859 - f1_score: 0.6898 - balanced_accuracy: 0.7953 - specificity: 0.8750 - miss_rate: 0.2844 - fall_out: 0.1250 - mcc: 0.5772 - val_loss: 0.3288 - val_accuracy: 0.8451 - val_recall: 0.7479 - val_precision: 0.6824 - val_AUROC: 0.9077 - val_AUPRC: 0.7185 - val_f1_score: 0.7137 - val_balanced_accuracy: 0.8134 - val_specificity: 0.8789 - val_miss_rate: 0.2521 - val_fall_out: 0.1211 - val_mcc: 0.6090
Epoch 3/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3489 - accuracy: 0.8377 - recall: 0.7167 - precision: 0.6748 - AUROC: 0.8966 - AUPRC: 0.6996 - f1_score: 0.6951 - balanced_accuracy: 0.7983 - specificity: 0.8799 - miss_rate: 0.2833 - fall_out: 0.1201 - mcc: 0.5852 - val_loss: 0.3260 - val_accuracy: 0.8470 - val_recall: 0.7252 - val_precision: 0.6952 - val_AUROC: 0.9098 - val_AUPRC: 0.7258 - val_f1_score: 0.7099 - val_balanced_accuracy: 0.8073 - val_specificity: 0.8893 - val_miss_rate: 0.2748 - val_fall_out: 0.1107 - val_mcc: 0.6063
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3432 - accuracy: 0.8402 - recall: 0.7139 - precision: 0.6819 - AUROC: 0.9001 - AUPRC: 0.7096 - f1_score: 0.6975 - balanced_accuracy: 0.7990 - specificity: 0.8841 - miss_rate: 0.2861 - fall_out: 0.1159 - mcc: 0.5893 - val_loss: 0.3268 - val_accuracy: 0.8464 - val_recall: 0.7904 - val_precision: 0.6722 - val_AUROC: 0.9094 - val_AUPRC: 0.7218 - val_f1_score: 0.7265 - val_balanced_accuracy: 0.8281 - val_specificity: 0.8659 - val_miss_rate: 0.2096 - val_fall_out: 0.1341 - val_mcc: 0.6246
Epoch 5/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3393 - accuracy: 0.8427 - recall: 0.7152 - precision: 0.6879 - AUROC: 0.9024 - AUPRC: 0.7117 - f1_score: 0.7013 - balanced_accuracy: 0.8011 - specificity: 0.8871 - miss_rate: 0.2848 - fall_out: 0.1129 - mcc: 0.5948 - val_loss: 0.3216 - val_accuracy: 0.8489 - val_recall: 0.7411 - val_precision: 0.6943 - val_AUROC: 0.9123 - val_AUPRC: 0.7342 - val_f1_score: 0.7169 - val_balanced_accuracy: 0.8138 - val_specificity: 0.8864 - val_miss_rate: 0.2589 - val_fall_out: 0.1136 - val_mcc: 0.6147
Epoch 6/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3351 - accuracy: 0.8446 - recall: 0.7137 - precision: 0.6933 - AUROC: 0.9046 - AUPRC: 0.7202 - f1_score: 0.7033 - balanced_accuracy: 0.8019 - specificity: 0.8901 - miss_rate: 0.2863 - fall_out: 0.1099 - mcc: 0.5982 - val_loss: 0.3201 - val_accuracy: 0.8500 - val_recall: 0.7338 - val_precision: 0.6998 - val_AUROC: 0.9130 - val_AUPRC: 0.7371 - val_f1_score: 0.7164 - val_balanced_accuracy: 0.8121 - val_specificity: 0.8905 - val_miss_rate: 0.2662 - val_fall_out: 0.1095 - val_mcc: 0.6149
Epoch 7/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3324 - accuracy: 0.8455 - recall: 0.7179 - precision: 0.6941 - AUROC: 0.9063 - AUPRC: 0.7282 - f1_score: 0.7058 - balanced_accuracy: 0.8039 - specificity: 0.8899 - miss_rate: 0.2821 - fall_out: 0.1101 - mcc: 0.6013 - val_loss: 0.3207 - val_accuracy: 0.8478 - val_recall: 0.7006 - val_precision: 0.7070 - val_AUROC: 0.9137 - val_AUPRC: 0.7401 - val_f1_score: 0.7038 - val_balanced_accuracy: 0.7998 - val_specificity: 0.8990 - val_miss_rate: 0.2994 - val_fall_out: 0.1010 - val_mcc: 0.6014
Epoch 8/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3319 - accuracy: 0.8460 - recall: 0.7105 - precision: 0.6981 - AUROC: 0.9065 - AUPRC: 0.7272 - f1_score: 0.7043 - balanced_accuracy: 0.8018 - specificity: 0.8931 - miss_rate: 0.2895 - fall_out: 0.1069 - mcc: 0.6002 - val_loss: 0.3235 - val_accuracy: 0.8477 - val_recall: 0.8098 - val_precision: 0.6696 - val_AUROC: 0.9124 - val_AUPRC: 0.7322 - val_f1_score: 0.7330 - val_balanced_accuracy: 0.8354 - val_specificity: 0.8609 - val_miss_rate: 0.1902 - val_fall_out: 0.1391 - val_mcc: 0.6334
-- HOLDOUT 5 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_4\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_26 (Dense) (None, 8) 4504
dropout_13 (Dropout) (None, 8) 0
dense_27 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_4\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 104, 'dropout_0': 0.3, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 8, 'dropout_2': 0.5, 'units_3': 136, 'dropout_3': 0.5, 'units_4': 104, 'dropout_4': 0.5}
-- 3 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'SRSF9', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'SRSF9', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_28 (Dense) (None, 104) 58136
dropout_14 (Dropout) (None, 104) 0
dense_29 (Dense) (None, 1) 105
=================================================================
Total params: 58,241
Trainable params: 58,241
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 8s 21ms/step - loss: 0.4092 - accuracy: 0.8140 - recall: 0.6826 - precision: 0.6287 - AUROC: 0.8662 - AUPRC: 0.6304 - f1_score: 0.6545 - balanced_accuracy: 0.7712 - specificity: 0.8597 - miss_rate: 0.3174 - fall_out: 0.1403 - mcc: 0.5284 - val_loss: 0.3534 - val_accuracy: 0.8374 - val_recall: 0.7811 - val_precision: 0.6553 - val_AUROC: 0.8975 - val_AUPRC: 0.6984 - val_f1_score: 0.7127 - val_balanced_accuracy: 0.8190 - val_specificity: 0.8570 - val_miss_rate: 0.2189 - val_fall_out: 0.1430 - val_mcc: 0.6050
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3437 - accuracy: 0.8419 - recall: 0.7307 - precision: 0.6804 - AUROC: 0.8996 - AUPRC: 0.7028 - f1_score: 0.7047 - balanced_accuracy: 0.8057 - specificity: 0.8806 - miss_rate: 0.2693 - fall_out: 0.1194 - mcc: 0.5976 - val_loss: 0.3385 - val_accuracy: 0.8431 - val_recall: 0.7782 - val_precision: 0.6685 - val_AUROC: 0.9048 - val_AUPRC: 0.7152 - val_f1_score: 0.7192 - val_balanced_accuracy: 0.8219 - val_specificity: 0.8657 - val_miss_rate: 0.2218 - val_fall_out: 0.1343 - val_mcc: 0.6146
Epoch 3/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3306 - accuracy: 0.8474 - recall: 0.7381 - precision: 0.6916 - AUROC: 0.9069 - AUPRC: 0.7270 - f1_score: 0.7141 - balanced_accuracy: 0.8118 - specificity: 0.8855 - miss_rate: 0.2619 - fall_out: 0.1145 - mcc: 0.6108 - val_loss: 0.3309 - val_accuracy: 0.8457 - val_recall: 0.7217 - val_precision: 0.6932 - val_AUROC: 0.9067 - val_AUPRC: 0.7243 - val_f1_score: 0.7072 - val_balanced_accuracy: 0.8053 - val_specificity: 0.8889 - val_miss_rate: 0.2783 - val_fall_out: 0.1111 - val_mcc: 0.6028
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3242 - accuracy: 0.8501 - recall: 0.7435 - precision: 0.6963 - AUROC: 0.9107 - AUPRC: 0.7368 - f1_score: 0.7191 - balanced_accuracy: 0.8153 - specificity: 0.8871 - miss_rate: 0.2565 - fall_out: 0.1129 - mcc: 0.6176 - val_loss: 0.3404 - val_accuracy: 0.8415 - val_recall: 0.8144 - val_precision: 0.6553 - val_AUROC: 0.9075 - val_AUPRC: 0.7219 - val_f1_score: 0.7263 - val_balanced_accuracy: 0.8327 - val_specificity: 0.8509 - val_miss_rate: 0.1856 - val_fall_out: 0.1491 - val_mcc: 0.6238
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3183 - accuracy: 0.8528 - recall: 0.7459 - precision: 0.7024 - AUROC: 0.9140 - AUPRC: 0.7490 - f1_score: 0.7235 - balanced_accuracy: 0.8180 - specificity: 0.8900 - miss_rate: 0.2541 - fall_out: 0.1100 - mcc: 0.6239 - val_loss: 0.3542 - val_accuracy: 0.8338 - val_recall: 0.7499 - val_precision: 0.6558 - val_AUROC: 0.8955 - val_AUPRC: 0.7057 - val_f1_score: 0.6997 - val_balanced_accuracy: 0.8064 - val_specificity: 0.8630 - val_miss_rate: 0.2501 - val_fall_out: 0.1370 - val_mcc: 0.5880
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 558
Rejected: 0
Iteration: 8 / 100
Confirmed: 409
Tentative: 149
Rejected: 0
Iteration: 9 / 100
Confirmed: 409
Tentative: 149
Rejected: 0
Iteration: 10 / 100
Confirmed: 409
Tentative: 149
Rejected: 0
Iteration: 11 / 100
Confirmed: 409
Tentative: 149
Rejected: 0
Iteration: 12 / 100
Confirmed: 435
Tentative: 123
Rejected: 0
Iteration: 13 / 100
Confirmed: 435
Tentative: 123
Rejected: 0
Iteration: 14 / 100
Confirmed: 435
Tentative: 123
Rejected: 0
Iteration: 15 / 100
Confirmed: 435
Tentative: 123
Rejected: 0
Iteration: 16 / 100
Confirmed: 443
Tentative: 115
Rejected: 0
Iteration: 17 / 100
Confirmed: 443
Tentative: 113
Rejected: 2
Iteration: 18 / 100
Confirmed: 443
Tentative: 113
Rejected: 2
Iteration: 19 / 100
Confirmed: 454
Tentative: 102
Rejected: 2
Iteration: 20 / 100
Confirmed: 454
Tentative: 102
Rejected: 2
Iteration: 21 / 100
Confirmed: 454
Tentative: 100
Rejected: 4
Iteration: 22 / 100
Confirmed: 458
Tentative: 96
Rejected: 4
Iteration: 23 / 100
Confirmed: 458
Tentative: 96
Rejected: 4
Iteration: 24 / 100
Confirmed: 458
Tentative: 96
Rejected: 4
Iteration: 25 / 100
Confirmed: 458
Tentative: 96
Rejected: 4
Iteration: 26 / 100
Confirmed: 463
Tentative: 91
Rejected: 4
Iteration: 27 / 100
Confirmed: 463
Tentative: 91
Rejected: 4
Iteration: 28 / 100
Confirmed: 463
Tentative: 84
Rejected: 11
Iteration: 29 / 100
Confirmed: 466
Tentative: 81
Rejected: 11
Iteration: 30 / 100
Confirmed: 466
Tentative: 81
Rejected: 11
Iteration: 31 / 100
Confirmed: 466
Tentative: 81
Rejected: 11
Iteration: 32 / 100
Confirmed: 467
Tentative: 80
Rejected: 11
Iteration: 33 / 100
Confirmed: 467
Tentative: 80
Rejected: 11
Iteration: 34 / 100
Confirmed: 468
Tentative: 79
Rejected: 11
Iteration: 35 / 100
Confirmed: 468
Tentative: 79
Rejected: 11
Iteration: 36 / 100
Confirmed: 468
Tentative: 79
Rejected: 11
Iteration: 37 / 100
Confirmed: 469
Tentative: 78
Rejected: 11
Iteration: 38 / 100
Confirmed: 469
Tentative: 78
Rejected: 11
Iteration: 39 / 100
Confirmed: 469
Tentative: 78
Rejected: 11
Iteration: 40 / 100
Confirmed: 471
Tentative: 76
Rejected: 11
Iteration: 41 / 100
Confirmed: 471
Tentative: 76
Rejected: 11
Iteration: 42 / 100
Confirmed: 471
Tentative: 76
Rejected: 11
Iteration: 43 / 100
Confirmed: 471
Tentative: 75
Rejected: 12
Iteration: 44 / 100
Confirmed: 471
Tentative: 75
Rejected: 12
Iteration: 45 / 100
Confirmed: 471
Tentative: 70
Rejected: 17
Iteration: 46 / 100
Confirmed: 471
Tentative: 70
Rejected: 17
Iteration: 47 / 100
Confirmed: 471
Tentative: 70
Rejected: 17
Iteration: 48 / 100
Confirmed: 471
Tentative: 70
Rejected: 17
Iteration: 49 / 100
Confirmed: 471
Tentative: 70
Rejected: 17
Iteration: 50 / 100
Confirmed: 471
Tentative: 68
Rejected: 19
Iteration: 51 / 100
Confirmed: 472
Tentative: 67
Rejected: 19
Iteration: 52 / 100
Confirmed: 472
Tentative: 67
Rejected: 19
Iteration: 53 / 100
Confirmed: 472
Tentative: 67
Rejected: 19
Iteration: 54 / 100
Confirmed: 475
Tentative: 64
Rejected: 19
Iteration: 55 / 100
Confirmed: 475
Tentative: 64
Rejected: 19
Iteration: 56 / 100
Confirmed: 475
Tentative: 64
Rejected: 19
Iteration: 57 / 100
Confirmed: 475
Tentative: 64
Rejected: 19
Iteration: 58 / 100
Confirmed: 475
Tentative: 64
Rejected: 19
Iteration: 59 / 100
Confirmed: 475
Tentative: 64
Rejected: 19
Iteration: 60 / 100
Confirmed: 475
Tentative: 63
Rejected: 20
Iteration: 61 / 100
Confirmed: 475
Tentative: 63
Rejected: 20
Iteration: 62 / 100
Confirmed: 475
Tentative: 62
Rejected: 21
Iteration: 63 / 100
Confirmed: 475
Tentative: 62
Rejected: 21
Iteration: 64 / 100
Confirmed: 475
Tentative: 62
Rejected: 21
Iteration: 65 / 100
Confirmed: 476
Tentative: 61
Rejected: 21
Iteration: 66 / 100
Confirmed: 476
Tentative: 61
Rejected: 21
Iteration: 67 / 100
Confirmed: 477
Tentative: 60
Rejected: 21
Iteration: 68 / 100
Confirmed: 477
Tentative: 60
Rejected: 21
Iteration: 69 / 100
Confirmed: 477
Tentative: 60
Rejected: 21
Iteration: 70 / 100
Confirmed: 478
Tentative: 59
Rejected: 21
Iteration: 71 / 100
Confirmed: 478
Tentative: 57
Rejected: 23
Iteration: 72 / 100
Confirmed: 481
Tentative: 54
Rejected: 23
Iteration: 73 / 100
Confirmed: 481
Tentative: 54
Rejected: 23
Iteration: 74 / 100
Confirmed: 481
Tentative: 54
Rejected: 23
Iteration: 75 / 100
Confirmed: 481
Tentative: 53
Rejected: 24
Iteration: 76 / 100
Confirmed: 481
Tentative: 53
Rejected: 24
Iteration: 77 / 100
Confirmed: 484
Tentative: 50
Rejected: 24
Iteration: 78 / 100
Confirmed: 484
Tentative: 50
Rejected: 24
Iteration: 79 / 100
Confirmed: 484
Tentative: 50
Rejected: 24
Iteration: 80 / 100
Confirmed: 484
Tentative: 49
Rejected: 25
Iteration: 81 / 100
Confirmed: 484
Tentative: 49
Rejected: 25
Iteration: 82 / 100
Confirmed: 484
Tentative: 49
Rejected: 25
Iteration: 83 / 100
Confirmed: 485
Tentative: 47
Rejected: 26
Iteration: 84 / 100
Confirmed: 485
Tentative: 47
Rejected: 26
Iteration: 85 / 100
Confirmed: 485
Tentative: 47
Rejected: 26
Iteration: 86 / 100
Confirmed: 485
Tentative: 47
Rejected: 26
Iteration: 87 / 100
Confirmed: 485
Tentative: 47
Rejected: 26
Iteration: 88 / 100
Confirmed: 486
Tentative: 46
Rejected: 26
Iteration: 89 / 100
Confirmed: 486
Tentative: 46
Rejected: 26
Iteration: 90 / 100
Confirmed: 486
Tentative: 46
Rejected: 26
Iteration: 91 / 100
Confirmed: 486
Tentative: 46
Rejected: 26
Iteration: 92 / 100
Confirmed: 486
Tentative: 46
Rejected: 26
Iteration: 93 / 100
Confirmed: 486
Tentative: 46
Rejected: 26
Iteration: 94 / 100
Confirmed: 486
Tentative: 46
Rejected: 26
Iteration: 95 / 100
Confirmed: 487
Tentative: 45
Rejected: 26
Iteration: 96 / 100
Confirmed: 487
Tentative: 45
Rejected: 26
Iteration: 97 / 100
Confirmed: 487
Tentative: 45
Rejected: 26
Iteration: 98 / 100
Confirmed: 487
Tentative: 45
Rejected: 26
Iteration: 99 / 100
Confirmed: 487
Tentative: 45
Rejected: 26
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 487
Tentative: 26
Rejected: 26
-- 487 Kept (correlated) features [Boruta]
-- 71 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_30 (Dense) (None, 104) 50752
dropout_15 (Dropout) (None, 104) 0
dense_31 (Dense) (None, 1) 105
=================================================================
Total params: 50,857
Trainable params: 50,857
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3890 - accuracy: 0.8255 - recall: 0.6894 - precision: 0.6536 - AUROC: 0.8775 - AUPRC: 0.6563 - f1_score: 0.6710 - balanced_accuracy: 0.7811 - specificity: 0.8729 - miss_rate: 0.3106 - fall_out: 0.1271 - mcc: 0.5528 - val_loss: 0.3435 - val_accuracy: 0.8364 - val_recall: 0.6482 - val_precision: 0.6969 - val_AUROC: 0.9002 - val_AUPRC: 0.7081 - val_f1_score: 0.6717 - val_balanced_accuracy: 0.7751 - val_specificity: 0.9019 - val_miss_rate: 0.3518 - val_fall_out: 0.0981 - val_mcc: 0.5636
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3393 - accuracy: 0.8409 - recall: 0.7312 - precision: 0.6777 - AUROC: 0.9016 - AUPRC: 0.7103 - f1_score: 0.7035 - balanced_accuracy: 0.8051 - specificity: 0.8790 - miss_rate: 0.2688 - fall_out: 0.1210 - mcc: 0.5957 - val_loss: 0.3330 - val_accuracy: 0.8436 - val_recall: 0.7603 - val_precision: 0.6750 - val_AUROC: 0.9056 - val_AUPRC: 0.7193 - val_f1_score: 0.7151 - val_balanced_accuracy: 0.8165 - val_specificity: 0.8726 - val_miss_rate: 0.2397 - val_fall_out: 0.1274 - val_mcc: 0.6099
Epoch 3/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3293 - accuracy: 0.8465 - recall: 0.7361 - precision: 0.6900 - AUROC: 0.9079 - AUPRC: 0.7290 - f1_score: 0.7123 - balanced_accuracy: 0.8105 - specificity: 0.8850 - miss_rate: 0.2639 - fall_out: 0.1150 - mcc: 0.6084 - val_loss: 0.3290 - val_accuracy: 0.8442 - val_recall: 0.7584 - val_precision: 0.6770 - val_AUROC: 0.9078 - val_AUPRC: 0.7264 - val_f1_score: 0.7154 - val_balanced_accuracy: 0.8162 - val_specificity: 0.8741 - val_miss_rate: 0.2416 - val_fall_out: 0.1259 - val_mcc: 0.6105
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3223 - accuracy: 0.8519 - recall: 0.7448 - precision: 0.7004 - AUROC: 0.9119 - AUPRC: 0.7410 - f1_score: 0.7219 - balanced_accuracy: 0.8170 - specificity: 0.8892 - miss_rate: 0.2552 - fall_out: 0.1108 - mcc: 0.6216 - val_loss: 0.3284 - val_accuracy: 0.8462 - val_recall: 0.6971 - val_precision: 0.7042 - val_AUROC: 0.9091 - val_AUPRC: 0.7288 - val_f1_score: 0.7006 - val_balanced_accuracy: 0.7976 - val_specificity: 0.8981 - val_miss_rate: 0.3029 - val_fall_out: 0.1019 - val_mcc: 0.5972
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3173 - accuracy: 0.8539 - recall: 0.7460 - precision: 0.7050 - AUROC: 0.9148 - AUPRC: 0.7492 - f1_score: 0.7249 - balanced_accuracy: 0.8187 - specificity: 0.8914 - miss_rate: 0.2540 - fall_out: 0.1086 - mcc: 0.6260 - val_loss: 0.3251 - val_accuracy: 0.8500 - val_recall: 0.7074 - val_precision: 0.7103 - val_AUROC: 0.9111 - val_AUPRC: 0.7361 - val_f1_score: 0.7088 - val_balanced_accuracy: 0.8035 - val_specificity: 0.8996 - val_miss_rate: 0.2926 - val_fall_out: 0.1004 - val_mcc: 0.6078
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3131 - accuracy: 0.8547 - recall: 0.7437 - precision: 0.7079 - AUROC: 0.9170 - AUPRC: 0.7572 - f1_score: 0.7254 - balanced_accuracy: 0.8185 - specificity: 0.8933 - miss_rate: 0.2563 - fall_out: 0.1067 - mcc: 0.6270 - val_loss: 0.3341 - val_accuracy: 0.8443 - val_recall: 0.8010 - val_precision: 0.6647 - val_AUROC: 0.9105 - val_AUPRC: 0.7312 - val_f1_score: 0.7265 - val_balanced_accuracy: 0.8302 - val_specificity: 0.8594 - val_miss_rate: 0.1990 - val_fall_out: 0.1406 - val_mcc: 0.6243
Epoch 7/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3093 - accuracy: 0.8567 - recall: 0.7520 - precision: 0.7101 - AUROC: 0.9194 - AUPRC: 0.7634 - f1_score: 0.7304 - balanced_accuracy: 0.8226 - specificity: 0.8932 - miss_rate: 0.2480 - fall_out: 0.1068 - mcc: 0.6335 - val_loss: 0.3261 - val_accuracy: 0.8457 - val_recall: 0.7848 - val_precision: 0.6723 - val_AUROC: 0.9103 - val_AUPRC: 0.7318 - val_f1_score: 0.7242 - val_balanced_accuracy: 0.8258 - val_specificity: 0.8669 - val_miss_rate: 0.2152 - val_fall_out: 0.1331 - val_mcc: 0.6215
-- HOLDOUT 6 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_5\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_32 (Dense) (None, 8) 4504
dropout_16 (Dropout) (None, 8) 0
dense_33 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_5\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 40, 'dropout_1': 0.5, 'units_2': 40, 'dropout_2': 0.3}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_34 (Dense) (None, 136) 76160
dropout_17 (Dropout) (None, 136) 0
dense_35 (Dense) (None, 1) 137
=================================================================
Total params: 76,297
Trainable params: 76,297
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 8s 21ms/step - loss: 0.4096 - accuracy: 0.8176 - recall: 0.6783 - precision: 0.6379 - AUROC: 0.8694 - AUPRC: 0.6345 - f1_score: 0.6575 - balanced_accuracy: 0.7722 - specificity: 0.8660 - miss_rate: 0.3217 - fall_out: 0.1340 - mcc: 0.5338 - val_loss: 0.3468 - val_accuracy: 0.8386 - val_recall: 0.7621 - val_precision: 0.6630 - val_AUROC: 0.8993 - val_AUPRC: 0.7061 - val_f1_score: 0.7091 - val_balanced_accuracy: 0.8136 - val_specificity: 0.8652 - val_miss_rate: 0.2379 - val_fall_out: 0.1348 - val_mcc: 0.6009
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3463 - accuracy: 0.8393 - recall: 0.7282 - precision: 0.6749 - AUROC: 0.8980 - AUPRC: 0.6995 - f1_score: 0.7005 - balanced_accuracy: 0.8031 - specificity: 0.8779 - miss_rate: 0.2718 - fall_out: 0.1221 - mcc: 0.5917 - val_loss: 0.3360 - val_accuracy: 0.8454 - val_recall: 0.7262 - val_precision: 0.6907 - val_AUROC: 0.9042 - val_AUPRC: 0.7202 - val_f1_score: 0.7080 - val_balanced_accuracy: 0.8065 - val_specificity: 0.8868 - val_miss_rate: 0.2738 - val_fall_out: 0.1132 - val_mcc: 0.6033
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3358 - accuracy: 0.8438 - recall: 0.7302 - precision: 0.6854 - AUROC: 0.9040 - AUPRC: 0.7177 - f1_score: 0.7071 - balanced_accuracy: 0.8068 - specificity: 0.8834 - miss_rate: 0.2698 - fall_out: 0.1166 - mcc: 0.6013 - val_loss: 0.3285 - val_accuracy: 0.8455 - val_recall: 0.7658 - val_precision: 0.6776 - val_AUROC: 0.9080 - val_AUPRC: 0.7307 - val_f1_score: 0.7190 - val_balanced_accuracy: 0.8195 - val_specificity: 0.8732 - val_miss_rate: 0.2342 - val_fall_out: 0.1268 - val_mcc: 0.6151
Epoch 4/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3286 - accuracy: 0.8481 - recall: 0.7374 - precision: 0.6936 - AUROC: 0.9080 - AUPRC: 0.7283 - f1_score: 0.7148 - balanced_accuracy: 0.8120 - specificity: 0.8867 - miss_rate: 0.2626 - fall_out: 0.1133 - mcc: 0.6120 - val_loss: 0.3267 - val_accuracy: 0.8473 - val_recall: 0.7632 - val_precision: 0.6826 - val_AUROC: 0.9096 - val_AUPRC: 0.7346 - val_f1_score: 0.7207 - val_balanced_accuracy: 0.8199 - val_specificity: 0.8765 - val_miss_rate: 0.2368 - val_fall_out: 0.1235 - val_mcc: 0.6179
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3260 - accuracy: 0.8487 - recall: 0.7350 - precision: 0.6958 - AUROC: 0.9096 - AUPRC: 0.7324 - f1_score: 0.7149 - balanced_accuracy: 0.8116 - specificity: 0.8882 - miss_rate: 0.2650 - fall_out: 0.1118 - mcc: 0.6124 - val_loss: 0.3274 - val_accuracy: 0.8466 - val_recall: 0.7020 - val_precision: 0.7033 - val_AUROC: 0.9091 - val_AUPRC: 0.7341 - val_f1_score: 0.7026 - val_balanced_accuracy: 0.7995 - val_specificity: 0.8970 - val_miss_rate: 0.2980 - val_fall_out: 0.1030 - val_mcc: 0.5993
Epoch 6/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3224 - accuracy: 0.8502 - recall: 0.7387 - precision: 0.6983 - AUROC: 0.9116 - AUPRC: 0.7371 - f1_score: 0.7179 - balanced_accuracy: 0.8138 - specificity: 0.8890 - miss_rate: 0.2613 - fall_out: 0.1110 - mcc: 0.6165 - val_loss: 0.3269 - val_accuracy: 0.8461 - val_recall: 0.7307 - val_precision: 0.6909 - val_AUROC: 0.9089 - val_AUPRC: 0.7324 - val_f1_score: 0.7102 - val_balanced_accuracy: 0.8084 - val_specificity: 0.8862 - val_miss_rate: 0.2693 - val_fall_out: 0.1138 - val_mcc: 0.6060
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 8 / 100
Confirmed: 414
Tentative: 145
Rejected: 0
Iteration: 9 / 100
Confirmed: 414
Tentative: 145
Rejected: 0
Iteration: 10 / 100
Confirmed: 414
Tentative: 145
Rejected: 0
Iteration: 11 / 100
Confirmed: 414
Tentative: 145
Rejected: 0
Iteration: 12 / 100
Confirmed: 435
Tentative: 120
Rejected: 4
Iteration: 13 / 100
Confirmed: 435
Tentative: 120
Rejected: 4
Iteration: 14 / 100
Confirmed: 435
Tentative: 120
Rejected: 4
Iteration: 15 / 100
Confirmed: 435
Tentative: 120
Rejected: 4
Iteration: 16 / 100
Confirmed: 442
Tentative: 113
Rejected: 4
Iteration: 17 / 100
Confirmed: 442
Tentative: 111
Rejected: 6
Iteration: 18 / 100
Confirmed: 442
Tentative: 111
Rejected: 6
Iteration: 19 / 100
Confirmed: 453
Tentative: 100
Rejected: 6
Iteration: 20 / 100
Confirmed: 453
Tentative: 100
Rejected: 6
Iteration: 21 / 100
Confirmed: 453
Tentative: 100
Rejected: 6
Iteration: 22 / 100
Confirmed: 453
Tentative: 100
Rejected: 6
Iteration: 23 / 100
Confirmed: 453
Tentative: 100
Rejected: 6
Iteration: 24 / 100
Confirmed: 453
Tentative: 98
Rejected: 8
Iteration: 25 / 100
Confirmed: 453
Tentative: 98
Rejected: 8
Iteration: 26 / 100
Confirmed: 454
Tentative: 97
Rejected: 8
Iteration: 27 / 100
Confirmed: 454
Tentative: 97
Rejected: 8
Iteration: 28 / 100
Confirmed: 454
Tentative: 97
Rejected: 8
Iteration: 29 / 100
Confirmed: 455
Tentative: 96
Rejected: 8
Iteration: 30 / 100
Confirmed: 455
Tentative: 94
Rejected: 10
Iteration: 31 / 100
Confirmed: 455
Tentative: 94
Rejected: 10
Iteration: 32 / 100
Confirmed: 459
Tentative: 90
Rejected: 10
Iteration: 33 / 100
Confirmed: 459
Tentative: 90
Rejected: 10
Iteration: 34 / 100
Confirmed: 462
Tentative: 87
Rejected: 10
Iteration: 35 / 100
Confirmed: 462
Tentative: 86
Rejected: 11
Iteration: 36 / 100
Confirmed: 462
Tentative: 86
Rejected: 11
Iteration: 37 / 100
Confirmed: 465
Tentative: 83
Rejected: 11
Iteration: 38 / 100
Confirmed: 465
Tentative: 82
Rejected: 12
Iteration: 39 / 100
Confirmed: 465
Tentative: 82
Rejected: 12
Iteration: 40 / 100
Confirmed: 468
Tentative: 79
Rejected: 12
Iteration: 41 / 100
Confirmed: 468
Tentative: 79
Rejected: 12
Iteration: 42 / 100
Confirmed: 468
Tentative: 79
Rejected: 12
Iteration: 43 / 100
Confirmed: 471
Tentative: 76
Rejected: 12
Iteration: 44 / 100
Confirmed: 471
Tentative: 76
Rejected: 12
Iteration: 45 / 100
Confirmed: 471
Tentative: 76
Rejected: 12
Iteration: 46 / 100
Confirmed: 472
Tentative: 75
Rejected: 12
Iteration: 47 / 100
Confirmed: 472
Tentative: 75
Rejected: 12
Iteration: 48 / 100
Confirmed: 472
Tentative: 75
Rejected: 12
Iteration: 49 / 100
Confirmed: 473
Tentative: 73
Rejected: 13
Iteration: 50 / 100
Confirmed: 473
Tentative: 73
Rejected: 13
Iteration: 51 / 100
Confirmed: 473
Tentative: 73
Rejected: 13
Iteration: 52 / 100
Confirmed: 473
Tentative: 73
Rejected: 13
Iteration: 53 / 100
Confirmed: 473
Tentative: 73
Rejected: 13
Iteration: 54 / 100
Confirmed: 474
Tentative: 71
Rejected: 14
Iteration: 55 / 100
Confirmed: 474
Tentative: 71
Rejected: 14
Iteration: 56 / 100
Confirmed: 474
Tentative: 71
Rejected: 14
Iteration: 57 / 100
Confirmed: 475
Tentative: 69
Rejected: 15
Iteration: 58 / 100
Confirmed: 475
Tentative: 69
Rejected: 15
Iteration: 59 / 100
Confirmed: 476
Tentative: 68
Rejected: 15
Iteration: 60 / 100
Confirmed: 476
Tentative: 67
Rejected: 16
Iteration: 61 / 100
Confirmed: 476
Tentative: 67
Rejected: 16
Iteration: 62 / 100
Confirmed: 478
Tentative: 65
Rejected: 16
Iteration: 63 / 100
Confirmed: 478
Tentative: 65
Rejected: 16
Iteration: 64 / 100
Confirmed: 478
Tentative: 65
Rejected: 16
Iteration: 65 / 100
Confirmed: 479
Tentative: 64
Rejected: 16
Iteration: 66 / 100
Confirmed: 479
Tentative: 64
Rejected: 16
Iteration: 67 / 100
Confirmed: 480
Tentative: 55
Rejected: 24
Iteration: 68 / 100
Confirmed: 480
Tentative: 55
Rejected: 24
Iteration: 69 / 100
Confirmed: 480
Tentative: 55
Rejected: 24
Iteration: 70 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 71 / 100
Confirmed: 481
Tentative: 54
Rejected: 24
Iteration: 72 / 100
Confirmed: 482
Tentative: 53
Rejected: 24
Iteration: 73 / 100
Confirmed: 482
Tentative: 53
Rejected: 24
Iteration: 74 / 100
Confirmed: 482
Tentative: 53
Rejected: 24
Iteration: 75 / 100
Confirmed: 482
Tentative: 53
Rejected: 24
Iteration: 76 / 100
Confirmed: 482
Tentative: 53
Rejected: 24
Iteration: 77 / 100
Confirmed: 482
Tentative: 53
Rejected: 24
Iteration: 78 / 100
Confirmed: 482
Tentative: 53
Rejected: 24
Iteration: 79 / 100
Confirmed: 482
Tentative: 53
Rejected: 24
Iteration: 80 / 100
Confirmed: 482
Tentative: 52
Rejected: 25
Iteration: 81 / 100
Confirmed: 482
Tentative: 52
Rejected: 25
Iteration: 82 / 100
Confirmed: 482
Tentative: 52
Rejected: 25
Iteration: 83 / 100
Confirmed: 482
Tentative: 51
Rejected: 26
Iteration: 84 / 100
Confirmed: 482
Tentative: 51
Rejected: 26
Iteration: 85 / 100
Confirmed: 482
Tentative: 51
Rejected: 26
Iteration: 86 / 100
Confirmed: 482
Tentative: 51
Rejected: 26
Iteration: 87 / 100
Confirmed: 482
Tentative: 51
Rejected: 26
Iteration: 88 / 100
Confirmed: 483
Tentative: 50
Rejected: 26
Iteration: 89 / 100
Confirmed: 483
Tentative: 50
Rejected: 26
Iteration: 90 / 100
Confirmed: 483
Tentative: 50
Rejected: 26
Iteration: 91 / 100
Confirmed: 483
Tentative: 50
Rejected: 26
Iteration: 92 / 100
Confirmed: 483
Tentative: 50
Rejected: 26
Iteration: 93 / 100
Confirmed: 483
Tentative: 50
Rejected: 26
Iteration: 94 / 100
Confirmed: 483
Tentative: 50
Rejected: 26
Iteration: 95 / 100
Confirmed: 484
Tentative: 49
Rejected: 26
Iteration: 96 / 100
Confirmed: 484
Tentative: 49
Rejected: 26
Iteration: 97 / 100
Confirmed: 484
Tentative: 49
Rejected: 26
Iteration: 98 / 100
Confirmed: 484
Tentative: 49
Rejected: 26
Iteration: 99 / 100
Confirmed: 484
Tentative: 49
Rejected: 26
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 484
Tentative: 24
Rejected: 26
-- 484 Kept (correlated) features [Boruta]
-- 75 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_36 (Dense) (None, 136) 65960
dropout_18 (Dropout) (None, 136) 0
dense_37 (Dense) (None, 1) 137
=================================================================
Total params: 66,097
Trainable params: 66,097
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.4157 - accuracy: 0.8129 - recall: 0.6824 - precision: 0.6263 - AUROC: 0.8652 - AUPRC: 0.6294 - f1_score: 0.6532 - balanced_accuracy: 0.7704 - specificity: 0.8584 - miss_rate: 0.3176 - fall_out: 0.1416 - mcc: 0.5263 - val_loss: 0.3433 - val_accuracy: 0.8396 - val_recall: 0.7774 - val_precision: 0.6609 - val_AUROC: 0.9010 - val_AUPRC: 0.7086 - val_f1_score: 0.7144 - val_balanced_accuracy: 0.8193 - val_specificity: 0.8612 - val_miss_rate: 0.2226 - val_fall_out: 0.1388 - val_mcc: 0.6077
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3468 - accuracy: 0.8383 - recall: 0.7331 - precision: 0.6710 - AUROC: 0.8976 - AUPRC: 0.6996 - f1_score: 0.7007 - balanced_accuracy: 0.8040 - specificity: 0.8749 - miss_rate: 0.2669 - fall_out: 0.1251 - mcc: 0.5913 - val_loss: 0.3342 - val_accuracy: 0.8428 - val_recall: 0.7198 - val_precision: 0.6864 - val_AUROC: 0.9047 - val_AUPRC: 0.7198 - val_f1_score: 0.7027 - val_balanced_accuracy: 0.8027 - val_specificity: 0.8856 - val_miss_rate: 0.2802 - val_fall_out: 0.1144 - val_mcc: 0.5962
Epoch 3/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3376 - accuracy: 0.8426 - recall: 0.7349 - precision: 0.6808 - AUROC: 0.9025 - AUPRC: 0.7130 - f1_score: 0.7068 - balanced_accuracy: 0.8075 - specificity: 0.8801 - miss_rate: 0.2651 - fall_out: 0.1199 - mcc: 0.6003 - val_loss: 0.3339 - val_accuracy: 0.8431 - val_recall: 0.7978 - val_precision: 0.6629 - val_AUROC: 0.9061 - val_AUPRC: 0.7247 - val_f1_score: 0.7241 - val_balanced_accuracy: 0.8283 - val_specificity: 0.8588 - val_miss_rate: 0.2022 - val_fall_out: 0.1412 - val_mcc: 0.6209
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3304 - accuracy: 0.8470 - recall: 0.7376 - precision: 0.6908 - AUROC: 0.9071 - AUPRC: 0.7262 - f1_score: 0.7134 - balanced_accuracy: 0.8114 - specificity: 0.8851 - miss_rate: 0.2624 - fall_out: 0.1149 - mcc: 0.6099 - val_loss: 0.3273 - val_accuracy: 0.8450 - val_recall: 0.7068 - val_precision: 0.6969 - val_AUROC: 0.9088 - val_AUPRC: 0.7301 - val_f1_score: 0.7018 - val_balanced_accuracy: 0.7999 - val_specificity: 0.8930 - val_miss_rate: 0.2932 - val_fall_out: 0.1070 - val_mcc: 0.5971
Epoch 5/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3262 - accuracy: 0.8479 - recall: 0.7381 - precision: 0.6928 - AUROC: 0.9093 - AUPRC: 0.7324 - f1_score: 0.7147 - balanced_accuracy: 0.8121 - specificity: 0.8861 - miss_rate: 0.2619 - fall_out: 0.1139 - mcc: 0.6117 - val_loss: 0.3256 - val_accuracy: 0.8455 - val_recall: 0.7832 - val_precision: 0.6724 - val_AUROC: 0.9096 - val_AUPRC: 0.7337 - val_f1_score: 0.7236 - val_balanced_accuracy: 0.8252 - val_specificity: 0.8672 - val_miss_rate: 0.2168 - val_fall_out: 0.1328 - val_mcc: 0.6207
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3224 - accuracy: 0.8502 - recall: 0.7398 - precision: 0.6980 - AUROC: 0.9116 - AUPRC: 0.7399 - f1_score: 0.7183 - balanced_accuracy: 0.8142 - specificity: 0.8886 - miss_rate: 0.2602 - fall_out: 0.1114 - mcc: 0.6169 - val_loss: 0.3277 - val_accuracy: 0.8469 - val_recall: 0.7917 - val_precision: 0.6730 - val_AUROC: 0.9106 - val_AUPRC: 0.7348 - val_f1_score: 0.7275 - val_balanced_accuracy: 0.8289 - val_specificity: 0.8661 - val_miss_rate: 0.2083 - val_fall_out: 0.1339 - val_mcc: 0.6261
Epoch 7/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3200 - accuracy: 0.8520 - recall: 0.7428 - precision: 0.7015 - AUROC: 0.9133 - AUPRC: 0.7453 - f1_score: 0.7216 - balanced_accuracy: 0.8164 - specificity: 0.8900 - miss_rate: 0.2572 - fall_out: 0.1100 - mcc: 0.6214 - val_loss: 0.3234 - val_accuracy: 0.8467 - val_recall: 0.7755 - val_precision: 0.6773 - val_AUROC: 0.9109 - val_AUPRC: 0.7343 - val_f1_score: 0.7231 - val_balanced_accuracy: 0.8235 - val_specificity: 0.8715 - val_miss_rate: 0.2245 - val_fall_out: 0.1285 - val_mcc: 0.6204
Epoch 8/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3157 - accuracy: 0.8545 - recall: 0.7457 - precision: 0.7068 - AUROC: 0.9157 - AUPRC: 0.7507 - f1_score: 0.7257 - balanced_accuracy: 0.8190 - specificity: 0.8924 - miss_rate: 0.2543 - fall_out: 0.1076 - mcc: 0.6272 - val_loss: 0.3215 - val_accuracy: 0.8469 - val_recall: 0.7553 - val_precision: 0.6843 - val_AUROC: 0.9119 - val_AUPRC: 0.7411 - val_f1_score: 0.7180 - val_balanced_accuracy: 0.8170 - val_specificity: 0.8787 - val_miss_rate: 0.2447 - val_fall_out: 0.1213 - val_mcc: 0.6147
Epoch 9/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3144 - accuracy: 0.8547 - recall: 0.7484 - precision: 0.7061 - AUROC: 0.9163 - AUPRC: 0.7525 - f1_score: 0.7266 - balanced_accuracy: 0.8200 - specificity: 0.8916 - miss_rate: 0.2516 - fall_out: 0.1084 - mcc: 0.6283 - val_loss: 0.3213 - val_accuracy: 0.8488 - val_recall: 0.7514 - val_precision: 0.6904 - val_AUROC: 0.9123 - val_AUPRC: 0.7436 - val_f1_score: 0.7196 - val_balanced_accuracy: 0.8171 - val_specificity: 0.8827 - val_miss_rate: 0.2486 - val_fall_out: 0.1173 - val_mcc: 0.6174
Epoch 10/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3118 - accuracy: 0.8557 - recall: 0.7432 - precision: 0.7109 - AUROC: 0.9179 - AUPRC: 0.7575 - f1_score: 0.7267 - balanced_accuracy: 0.8190 - specificity: 0.8948 - miss_rate: 0.2568 - fall_out: 0.1052 - mcc: 0.6290 - val_loss: 0.3236 - val_accuracy: 0.8504 - val_recall: 0.7380 - val_precision: 0.6991 - val_AUROC: 0.9117 - val_AUPRC: 0.7409 - val_f1_score: 0.7180 - val_balanced_accuracy: 0.8137 - val_specificity: 0.8895 - val_miss_rate: 0.2620 - val_fall_out: 0.1105 - val_mcc: 0.6167
Epoch 11/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3102 - accuracy: 0.8581 - recall: 0.7504 - precision: 0.7144 - AUROC: 0.9188 - AUPRC: 0.7615 - f1_score: 0.7319 - balanced_accuracy: 0.8230 - specificity: 0.8956 - miss_rate: 0.2496 - fall_out: 0.1044 - mcc: 0.6359 - val_loss: 0.3206 - val_accuracy: 0.8497 - val_recall: 0.7043 - val_precision: 0.7108 - val_AUROC: 0.9126 - val_AUPRC: 0.7451 - val_f1_score: 0.7075 - val_balanced_accuracy: 0.8023 - val_specificity: 0.9003 - val_miss_rate: 0.2957 - val_fall_out: 0.0997 - val_mcc: 0.6064
Epoch 12/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3083 - accuracy: 0.8583 - recall: 0.7514 - precision: 0.7143 - AUROC: 0.9200 - AUPRC: 0.7655 - f1_score: 0.7324 - balanced_accuracy: 0.8234 - specificity: 0.8955 - miss_rate: 0.2486 - fall_out: 0.1045 - mcc: 0.6365 - val_loss: 0.3200 - val_accuracy: 0.8503 - val_recall: 0.7340 - val_precision: 0.7004 - val_AUROC: 0.9134 - val_AUPRC: 0.7424 - val_f1_score: 0.7168 - val_balanced_accuracy: 0.8124 - val_specificity: 0.8908 - val_miss_rate: 0.2660 - val_fall_out: 0.1092 - val_mcc: 0.6154
Epoch 13/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3057 - accuracy: 0.8595 - recall: 0.7520 - precision: 0.7172 - AUROC: 0.9212 - AUPRC: 0.7697 - f1_score: 0.7342 - balanced_accuracy: 0.8244 - specificity: 0.8968 - miss_rate: 0.2480 - fall_out: 0.1032 - mcc: 0.6391 - val_loss: 0.3206 - val_accuracy: 0.8509 - val_recall: 0.7795 - val_precision: 0.6858 - val_AUROC: 0.9133 - val_AUPRC: 0.7412 - val_f1_score: 0.7296 - val_balanced_accuracy: 0.8276 - val_specificity: 0.8757 - val_miss_rate: 0.2205 - val_fall_out: 0.1243 - val_mcc: 0.6297
Epoch 14/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3046 - accuracy: 0.8610 - recall: 0.7582 - precision: 0.7187 - AUROC: 0.9220 - AUPRC: 0.7699 - f1_score: 0.7379 - balanced_accuracy: 0.8275 - specificity: 0.8968 - miss_rate: 0.2418 - fall_out: 0.1032 - mcc: 0.6439 - val_loss: 0.3181 - val_accuracy: 0.8511 - val_recall: 0.7642 - val_precision: 0.6914 - val_AUROC: 0.9140 - val_AUPRC: 0.7428 - val_f1_score: 0.7260 - val_balanced_accuracy: 0.8228 - val_specificity: 0.8813 - val_miss_rate: 0.2358 - val_fall_out: 0.1187 - val_mcc: 0.6256
Epoch 15/1000
313/313 [==============================] - 6s 18ms/step - loss: 0.3020 - accuracy: 0.8613 - recall: 0.7568 - precision: 0.7201 - AUROC: 0.9232 - AUPRC: 0.7745 - f1_score: 0.7380 - balanced_accuracy: 0.8272 - specificity: 0.8976 - miss_rate: 0.2432 - fall_out: 0.1024 - mcc: 0.6441 - val_loss: 0.3219 - val_accuracy: 0.8495 - val_recall: 0.7931 - val_precision: 0.6784 - val_AUROC: 0.9138 - val_AUPRC: 0.7448 - val_f1_score: 0.7313 - val_balanced_accuracy: 0.8311 - val_specificity: 0.8692 - val_miss_rate: 0.2069 - val_fall_out: 0.1308 - val_mcc: 0.6314
Epoch 16/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3003 - accuracy: 0.8613 - recall: 0.7597 - precision: 0.7190 - AUROC: 0.9242 - AUPRC: 0.7789 - f1_score: 0.7388 - balanced_accuracy: 0.8282 - specificity: 0.8967 - miss_rate: 0.2403 - fall_out: 0.1033 - mcc: 0.6450 - val_loss: 0.3199 - val_accuracy: 0.8483 - val_recall: 0.6884 - val_precision: 0.7139 - val_AUROC: 0.9137 - val_AUPRC: 0.7450 - val_f1_score: 0.7009 - val_balanced_accuracy: 0.7962 - val_specificity: 0.9040 - val_miss_rate: 0.3116 - val_fall_out: 0.0960 - val_mcc: 0.5995
-- HOLDOUT 7 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_6\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_38 (Dense) (None, 8) 4504
dropout_19 (Dropout) (None, 8) 0
dense_39 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_6\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 136, 'dropout_2': 0.5, 'units_3': 136, 'dropout_3': 0.5, 'units_4': 136, 'dropout_4': 0.5}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_40 (Dense) (None, 136) 76160
dropout_20 (Dropout) (None, 136) 0
dense_41 (Dense) (None, 1) 137
=================================================================
Total params: 76,297
Trainable params: 76,297
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3950 - accuracy: 0.8206 - recall: 0.6871 - precision: 0.6425 - AUROC: 0.8732 - AUPRC: 0.6458 - f1_score: 0.6641 - balanced_accuracy: 0.7770 - specificity: 0.8670 - miss_rate: 0.3129 - fall_out: 0.1330 - mcc: 0.5424 - val_loss: 0.3461 - val_accuracy: 0.8377 - val_recall: 0.7931 - val_precision: 0.6528 - val_AUROC: 0.9002 - val_AUPRC: 0.7058 - val_f1_score: 0.7162 - val_balanced_accuracy: 0.8232 - val_specificity: 0.8532 - val_miss_rate: 0.2069 - val_fall_out: 0.1468 - val_mcc: 0.6096
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3380 - accuracy: 0.8436 - recall: 0.7311 - precision: 0.6844 - AUROC: 0.9031 - AUPRC: 0.7158 - f1_score: 0.7070 - balanced_accuracy: 0.8069 - specificity: 0.8827 - miss_rate: 0.2689 - fall_out: 0.1173 - mcc: 0.6010 - val_loss: 0.3350 - val_accuracy: 0.8440 - val_recall: 0.7460 - val_precision: 0.6805 - val_AUROC: 0.9052 - val_AUPRC: 0.7204 - val_f1_score: 0.7117 - val_balanced_accuracy: 0.8121 - val_specificity: 0.8781 - val_miss_rate: 0.2540 - val_fall_out: 0.1219 - val_mcc: 0.6063
Epoch 3/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3265 - accuracy: 0.8485 - recall: 0.7372 - precision: 0.6945 - AUROC: 0.9091 - AUPRC: 0.7341 - f1_score: 0.7152 - balanced_accuracy: 0.8122 - specificity: 0.8872 - miss_rate: 0.2628 - fall_out: 0.1128 - mcc: 0.6126 - val_loss: 0.3384 - val_accuracy: 0.8423 - val_recall: 0.8086 - val_precision: 0.6584 - val_AUROC: 0.9056 - val_AUPRC: 0.7202 - val_f1_score: 0.7258 - val_balanced_accuracy: 0.8313 - val_specificity: 0.8540 - val_miss_rate: 0.1914 - val_fall_out: 0.1460 - val_mcc: 0.6231
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3196 - accuracy: 0.8516 - recall: 0.7413 - precision: 0.7011 - AUROC: 0.9134 - AUPRC: 0.7458 - f1_score: 0.7206 - balanced_accuracy: 0.8157 - specificity: 0.8900 - miss_rate: 0.2587 - fall_out: 0.1100 - mcc: 0.6202 - val_loss: 0.3276 - val_accuracy: 0.8458 - val_recall: 0.7359 - val_precision: 0.6884 - val_AUROC: 0.9085 - val_AUPRC: 0.7300 - val_f1_score: 0.7113 - val_balanced_accuracy: 0.8100 - val_specificity: 0.8841 - val_miss_rate: 0.2641 - val_fall_out: 0.1159 - val_mcc: 0.6070
Epoch 5/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3142 - accuracy: 0.8555 - recall: 0.7485 - precision: 0.7082 - AUROC: 0.9164 - AUPRC: 0.7551 - f1_score: 0.7278 - balanced_accuracy: 0.8206 - specificity: 0.8927 - miss_rate: 0.2515 - fall_out: 0.1073 - mcc: 0.6299 - val_loss: 0.3349 - val_accuracy: 0.8392 - val_recall: 0.6240 - val_precision: 0.7165 - val_AUROC: 0.9067 - val_AUPRC: 0.7278 - val_f1_score: 0.6671 - val_balanced_accuracy: 0.7691 - val_specificity: 0.9141 - val_miss_rate: 0.3760 - val_fall_out: 0.0859 - val_mcc: 0.5641
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3104 - accuracy: 0.8570 - recall: 0.7491 - precision: 0.7120 - AUROC: 0.9185 - AUPRC: 0.7629 - f1_score: 0.7301 - balanced_accuracy: 0.8219 - specificity: 0.8946 - miss_rate: 0.2509 - fall_out: 0.1054 - mcc: 0.6333 - val_loss: 0.3434 - val_accuracy: 0.8421 - val_recall: 0.7345 - val_precision: 0.6797 - val_AUROC: 0.9019 - val_AUPRC: 0.7200 - val_f1_score: 0.7061 - val_balanced_accuracy: 0.8070 - val_specificity: 0.8796 - val_miss_rate: 0.2655 - val_fall_out: 0.1204 - val_mcc: 0.5992
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 8 / 100
Confirmed: 416
Tentative: 143
Rejected: 0
Iteration: 9 / 100
Confirmed: 416
Tentative: 143
Rejected: 0
Iteration: 10 / 100
Confirmed: 416
Tentative: 143
Rejected: 0
Iteration: 11 / 100
Confirmed: 416
Tentative: 137
Rejected: 6
Iteration: 12 / 100
Confirmed: 434
Tentative: 119
Rejected: 6
Iteration: 13 / 100
Confirmed: 434
Tentative: 119
Rejected: 6
Iteration: 14 / 100
Confirmed: 434
Tentative: 119
Rejected: 6
Iteration: 15 / 100
Confirmed: 434
Tentative: 119
Rejected: 6
Iteration: 16 / 100
Confirmed: 442
Tentative: 111
Rejected: 6
Iteration: 17 / 100
Confirmed: 442
Tentative: 111
Rejected: 6
Iteration: 18 / 100
Confirmed: 442
Tentative: 110
Rejected: 7
Iteration: 19 / 100
Confirmed: 451
Tentative: 101
Rejected: 7
Iteration: 20 / 100
Confirmed: 451
Tentative: 101
Rejected: 7
Iteration: 21 / 100
Confirmed: 451
Tentative: 101
Rejected: 7
Iteration: 22 / 100
Confirmed: 456
Tentative: 91
Rejected: 12
Iteration: 23 / 100
Confirmed: 456
Tentative: 91
Rejected: 12
Iteration: 24 / 100
Confirmed: 456
Tentative: 91
Rejected: 12
Iteration: 25 / 100
Confirmed: 456
Tentative: 91
Rejected: 12
Iteration: 26 / 100
Confirmed: 459
Tentative: 88
Rejected: 12
Iteration: 27 / 100
Confirmed: 459
Tentative: 86
Rejected: 14
Iteration: 28 / 100
Confirmed: 459
Tentative: 86
Rejected: 14
Iteration: 29 / 100
Confirmed: 462
Tentative: 83
Rejected: 14
Iteration: 30 / 100
Confirmed: 462
Tentative: 83
Rejected: 14
Iteration: 31 / 100
Confirmed: 462
Tentative: 83
Rejected: 14
Iteration: 32 / 100
Confirmed: 463
Tentative: 82
Rejected: 14
Iteration: 33 / 100
Confirmed: 463
Tentative: 82
Rejected: 14
Iteration: 34 / 100
Confirmed: 467
Tentative: 78
Rejected: 14
Iteration: 35 / 100
Confirmed: 467
Tentative: 78
Rejected: 14
Iteration: 36 / 100
Confirmed: 467
Tentative: 78
Rejected: 14
Iteration: 37 / 100
Confirmed: 470
Tentative: 75
Rejected: 14
Iteration: 38 / 100
Confirmed: 470
Tentative: 75
Rejected: 14
Iteration: 39 / 100
Confirmed: 470
Tentative: 75
Rejected: 14
Iteration: 40 / 100
Confirmed: 472
Tentative: 73
Rejected: 14
Iteration: 41 / 100
Confirmed: 472
Tentative: 70
Rejected: 17
Iteration: 42 / 100
Confirmed: 472
Tentative: 70
Rejected: 17
Iteration: 43 / 100
Confirmed: 474
Tentative: 68
Rejected: 17
Iteration: 44 / 100
Confirmed: 474
Tentative: 68
Rejected: 17
Iteration: 45 / 100
Confirmed: 474
Tentative: 68
Rejected: 17
Iteration: 46 / 100
Confirmed: 475
Tentative: 63
Rejected: 21
Iteration: 47 / 100
Confirmed: 475
Tentative: 63
Rejected: 21
Iteration: 48 / 100
Confirmed: 475
Tentative: 63
Rejected: 21
Iteration: 49 / 100
Confirmed: 476
Tentative: 62
Rejected: 21
Iteration: 50 / 100
Confirmed: 476
Tentative: 62
Rejected: 21
Iteration: 51 / 100
Confirmed: 478
Tentative: 60
Rejected: 21
Iteration: 52 / 100
Confirmed: 478
Tentative: 60
Rejected: 21
Iteration: 53 / 100
Confirmed: 478
Tentative: 60
Rejected: 21
Iteration: 54 / 100
Confirmed: 478
Tentative: 60
Rejected: 21
Iteration: 55 / 100
Confirmed: 478
Tentative: 60
Rejected: 21
Iteration: 56 / 100
Confirmed: 478
Tentative: 60
Rejected: 21
Iteration: 57 / 100
Confirmed: 478
Tentative: 59
Rejected: 22
Iteration: 58 / 100
Confirmed: 478
Tentative: 59
Rejected: 22
Iteration: 59 / 100
Confirmed: 479
Tentative: 58
Rejected: 22
Iteration: 60 / 100
Confirmed: 479
Tentative: 58
Rejected: 22
Iteration: 61 / 100
Confirmed: 479
Tentative: 58
Rejected: 22
Iteration: 62 / 100
Confirmed: 479
Tentative: 58
Rejected: 22
Iteration: 63 / 100
Confirmed: 479
Tentative: 58
Rejected: 22
Iteration: 64 / 100
Confirmed: 479
Tentative: 58
Rejected: 22
Iteration: 65 / 100
Confirmed: 479
Tentative: 58
Rejected: 22
Iteration: 66 / 100
Confirmed: 479
Tentative: 58
Rejected: 22
Iteration: 67 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 68 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 69 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 70 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 71 / 100
Confirmed: 480
Tentative: 52
Rejected: 27
Iteration: 72 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 73 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 74 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 75 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 76 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 77 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 78 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 79 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 80 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 81 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 82 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 83 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 84 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 85 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 86 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 87 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 88 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 89 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 90 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 91 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 92 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 93 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 94 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 95 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 96 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 97 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 98 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 99 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 483
Tentative: 30
Rejected: 28
-- 483 Kept (correlated) features [Boruta]
-- 76 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_42 (Dense) (None, 136) 65824
dropout_21 (Dropout) (None, 136) 0
dense_43 (Dense) (None, 1) 137
=================================================================
Total params: 65,961
Trainable params: 65,961
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3892 - accuracy: 0.8237 - recall: 0.6800 - precision: 0.6520 - AUROC: 0.8770 - AUPRC: 0.6559 - f1_score: 0.6657 - balanced_accuracy: 0.7769 - specificity: 0.8737 - miss_rate: 0.3200 - fall_out: 0.1263 - mcc: 0.5463 - val_loss: 0.3429 - val_accuracy: 0.8409 - val_recall: 0.7440 - val_precision: 0.6737 - val_AUROC: 0.9000 - val_AUPRC: 0.7010 - val_f1_score: 0.7072 - val_balanced_accuracy: 0.8093 - val_specificity: 0.8746 - val_miss_rate: 0.2560 - val_fall_out: 0.1254 - val_mcc: 0.5997
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3375 - accuracy: 0.8428 - recall: 0.7232 - precision: 0.6853 - AUROC: 0.9027 - AUPRC: 0.7136 - f1_score: 0.7037 - balanced_accuracy: 0.8038 - specificity: 0.8844 - miss_rate: 0.2768 - fall_out: 0.1156 - mcc: 0.5973 - val_loss: 0.3359 - val_accuracy: 0.8418 - val_recall: 0.7592 - val_precision: 0.6711 - val_AUROC: 0.9042 - val_AUPRC: 0.7099 - val_f1_score: 0.7124 - val_balanced_accuracy: 0.8148 - val_specificity: 0.8705 - val_miss_rate: 0.2408 - val_fall_out: 0.1295 - val_mcc: 0.6060
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3271 - accuracy: 0.8469 - recall: 0.7360 - precision: 0.6909 - AUROC: 0.9089 - AUPRC: 0.7300 - f1_score: 0.7128 - balanced_accuracy: 0.8107 - specificity: 0.8854 - miss_rate: 0.2640 - fall_out: 0.1146 - mcc: 0.6091 - val_loss: 0.3310 - val_accuracy: 0.8454 - val_recall: 0.7557 - val_precision: 0.6806 - val_AUROC: 0.9067 - val_AUPRC: 0.7218 - val_f1_score: 0.7162 - val_balanced_accuracy: 0.8161 - val_specificity: 0.8766 - val_miss_rate: 0.2443 - val_fall_out: 0.1234 - val_mcc: 0.6119
Epoch 4/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3212 - accuracy: 0.8516 - recall: 0.7426 - precision: 0.7006 - AUROC: 0.9122 - AUPRC: 0.7416 - f1_score: 0.7210 - balanced_accuracy: 0.8161 - specificity: 0.8896 - miss_rate: 0.2574 - fall_out: 0.1104 - mcc: 0.6206 - val_loss: 0.3338 - val_accuracy: 0.8424 - val_recall: 0.8020 - val_precision: 0.6603 - val_AUROC: 0.9080 - val_AUPRC: 0.7208 - val_f1_score: 0.7243 - val_balanced_accuracy: 0.8292 - val_specificity: 0.8564 - val_miss_rate: 0.1980 - val_fall_out: 0.1436 - val_mcc: 0.6211
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3155 - accuracy: 0.8540 - recall: 0.7474 - precision: 0.7049 - AUROC: 0.9155 - AUPRC: 0.7515 - f1_score: 0.7256 - balanced_accuracy: 0.8193 - specificity: 0.8911 - miss_rate: 0.2526 - fall_out: 0.1089 - mcc: 0.6268 - val_loss: 0.3259 - val_accuracy: 0.8470 - val_recall: 0.7609 - val_precision: 0.6828 - val_AUROC: 0.9095 - val_AUPRC: 0.7318 - val_f1_score: 0.7197 - val_balanced_accuracy: 0.8189 - val_specificity: 0.8770 - val_miss_rate: 0.2391 - val_fall_out: 0.1230 - val_mcc: 0.6167
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3116 - accuracy: 0.8555 - recall: 0.7487 - precision: 0.7083 - AUROC: 0.9179 - AUPRC: 0.7603 - f1_score: 0.7279 - balanced_accuracy: 0.8207 - specificity: 0.8927 - miss_rate: 0.2513 - fall_out: 0.1073 - mcc: 0.6301 - val_loss: 0.3457 - val_accuracy: 0.8463 - val_recall: 0.7793 - val_precision: 0.6753 - val_AUROC: 0.9045 - val_AUPRC: 0.7276 - val_f1_score: 0.7236 - val_balanced_accuracy: 0.8245 - val_specificity: 0.8696 - val_miss_rate: 0.2207 - val_fall_out: 0.1304 - val_mcc: 0.6210
Epoch 7/1000
313/313 [==============================] - 6s 20ms/step - loss: 0.3069 - accuracy: 0.8590 - recall: 0.7486 - precision: 0.7175 - AUROC: 0.9207 - AUPRC: 0.7671 - f1_score: 0.7327 - balanced_accuracy: 0.8230 - specificity: 0.8975 - miss_rate: 0.2514 - fall_out: 0.1025 - mcc: 0.6373 - val_loss: 0.3290 - val_accuracy: 0.8474 - val_recall: 0.7850 - val_precision: 0.6761 - val_AUROC: 0.9100 - val_AUPRC: 0.7337 - val_f1_score: 0.7265 - val_balanced_accuracy: 0.8271 - val_specificity: 0.8692 - val_miss_rate: 0.2150 - val_fall_out: 0.1308 - val_mcc: 0.6248
-- HOLDOUT 8 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_7\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_44 (Dense) (None, 8) 4504
dropout_22 (Dropout) (None, 8) 0
dense_45 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_7\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 4, 'units_0': 72, 'dropout_0': 0.5, 'units_1': 104, 'dropout_1': 0.4, 'units_2': 40, 'dropout_2': 0.5, 'units_3': 40, 'dropout_3': 0.3}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_46 (Dense) (None, 72) 40320
dropout_23 (Dropout) (None, 72) 0
dense_47 (Dense) (None, 104) 7592
dropout_24 (Dropout) (None, 104) 0
dense_48 (Dense) (None, 40) 4200
dropout_25 (Dropout) (None, 40) 0
dense_49 (Dense) (None, 40) 1640
dropout_26 (Dropout) (None, 40) 0
dense_50 (Dense) (None, 1) 41
=================================================================
Total params: 53,793
Trainable params: 53,793
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 10s 28ms/step - loss: 0.4533 - accuracy: 0.7697 - recall: 0.4688 - precision: 0.5648 - AUROC: 0.8224 - AUPRC: 0.5520 - f1_score: 0.5124 - balanced_accuracy: 0.6716 - specificity: 0.8743 - miss_rate: 0.5312 - fall_out: 0.1257 - mcc: 0.3660 - val_loss: 0.3606 - val_accuracy: 0.8290 - val_recall: 0.6391 - val_precision: 0.6794 - val_AUROC: 0.8952 - val_AUPRC: 0.6799 - val_f1_score: 0.6587 - val_balanced_accuracy: 0.7671 - val_specificity: 0.8951 - val_miss_rate: 0.3609 - val_fall_out: 0.1049 - val_mcc: 0.5452
Epoch 2/1000
313/313 [==============================] - 8s 27ms/step - loss: 0.3751 - accuracy: 0.8177 - recall: 0.6092 - precision: 0.6588 - AUROC: 0.8815 - AUPRC: 0.6603 - f1_score: 0.6330 - balanced_accuracy: 0.7497 - specificity: 0.8902 - miss_rate: 0.3908 - fall_out: 0.1098 - mcc: 0.5127 - val_loss: 0.3545 - val_accuracy: 0.8386 - val_recall: 0.6857 - val_precision: 0.6879 - val_AUROC: 0.9004 - val_AUPRC: 0.6958 - val_f1_score: 0.6868 - val_balanced_accuracy: 0.7887 - val_specificity: 0.8918 - val_miss_rate: 0.3143 - val_fall_out: 0.1082 - val_mcc: 0.5781
Epoch 3/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3606 - accuracy: 0.8279 - recall: 0.6857 - precision: 0.6604 - AUROC: 0.8889 - AUPRC: 0.6745 - f1_score: 0.6728 - balanced_accuracy: 0.7815 - specificity: 0.8773 - miss_rate: 0.3143 - fall_out: 0.1227 - mcc: 0.5563 - val_loss: 0.3611 - val_accuracy: 0.8401 - val_recall: 0.6713 - val_precision: 0.6977 - val_AUROC: 0.8992 - val_AUPRC: 0.7017 - val_f1_score: 0.6843 - val_balanced_accuracy: 0.7851 - val_specificity: 0.8988 - val_miss_rate: 0.3287 - val_fall_out: 0.1012 - val_mcc: 0.5774
Epoch 4/1000
313/313 [==============================] - 8s 27ms/step - loss: 0.3522 - accuracy: 0.8340 - recall: 0.7255 - precision: 0.6631 - AUROC: 0.8936 - AUPRC: 0.6848 - f1_score: 0.6929 - balanced_accuracy: 0.7986 - specificity: 0.8717 - miss_rate: 0.2745 - fall_out: 0.1283 - mcc: 0.5806 - val_loss: 0.3461 - val_accuracy: 0.8423 - val_recall: 0.7912 - val_precision: 0.6630 - val_AUROC: 0.9040 - val_AUPRC: 0.7049 - val_f1_score: 0.7214 - val_balanced_accuracy: 0.8256 - val_specificity: 0.8601 - val_miss_rate: 0.2088 - val_fall_out: 0.1399 - val_mcc: 0.6173
Epoch 5/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3481 - accuracy: 0.8383 - recall: 0.7468 - precision: 0.6667 - AUROC: 0.8958 - AUPRC: 0.6905 - f1_score: 0.7045 - balanced_accuracy: 0.8084 - specificity: 0.8701 - miss_rate: 0.2532 - fall_out: 0.1299 - mcc: 0.5955 - val_loss: 0.3456 - val_accuracy: 0.8417 - val_recall: 0.8183 - val_precision: 0.6547 - val_AUROC: 0.9040 - val_AUPRC: 0.7049 - val_f1_score: 0.7274 - val_balanced_accuracy: 0.8341 - val_specificity: 0.8498 - val_miss_rate: 0.1817 - val_fall_out: 0.1502 - val_mcc: 0.6254
Epoch 6/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3431 - accuracy: 0.8399 - recall: 0.7534 - precision: 0.6684 - AUROC: 0.8984 - AUPRC: 0.6974 - f1_score: 0.7083 - balanced_accuracy: 0.8117 - specificity: 0.8699 - miss_rate: 0.2466 - fall_out: 0.1301 - mcc: 0.6006 - val_loss: 0.3445 - val_accuracy: 0.8439 - val_recall: 0.7923 - val_precision: 0.6661 - val_AUROC: 0.9057 - val_AUPRC: 0.7110 - val_f1_score: 0.7238 - val_balanced_accuracy: 0.8271 - val_specificity: 0.8618 - val_miss_rate: 0.2077 - val_fall_out: 0.1382 - val_mcc: 0.6206
Epoch 7/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3408 - accuracy: 0.8419 - recall: 0.7599 - precision: 0.6711 - AUROC: 0.9001 - AUPRC: 0.7023 - f1_score: 0.7127 - balanced_accuracy: 0.8151 - specificity: 0.8704 - miss_rate: 0.2401 - fall_out: 0.1296 - mcc: 0.6064 - val_loss: 0.3433 - val_accuracy: 0.8448 - val_recall: 0.7714 - val_precision: 0.6742 - val_AUROC: 0.9067 - val_AUPRC: 0.7144 - val_f1_score: 0.7195 - val_balanced_accuracy: 0.8208 - val_specificity: 0.8703 - val_miss_rate: 0.2286 - val_fall_out: 0.1297 - val_mcc: 0.6156
Epoch 8/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3383 - accuracy: 0.8425 - recall: 0.7710 - precision: 0.6692 - AUROC: 0.9010 - AUPRC: 0.7002 - f1_score: 0.7165 - balanced_accuracy: 0.8192 - specificity: 0.8674 - miss_rate: 0.2290 - fall_out: 0.1326 - mcc: 0.6111 - val_loss: 0.3513 - val_accuracy: 0.8447 - val_recall: 0.7782 - val_precision: 0.6720 - val_AUROC: 0.9050 - val_AUPRC: 0.7108 - val_f1_score: 0.7212 - val_balanced_accuracy: 0.8230 - val_specificity: 0.8678 - val_miss_rate: 0.2218 - val_fall_out: 0.1322 - val_mcc: 0.6175
Epoch 9/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3356 - accuracy: 0.8441 - recall: 0.7704 - precision: 0.6730 - AUROC: 0.9029 - AUPRC: 0.7075 - f1_score: 0.7184 - balanced_accuracy: 0.8201 - specificity: 0.8698 - miss_rate: 0.2296 - fall_out: 0.1302 - mcc: 0.6140 - val_loss: 0.3413 - val_accuracy: 0.8448 - val_recall: 0.8041 - val_precision: 0.6648 - val_AUROC: 0.9073 - val_AUPRC: 0.7176 - val_f1_score: 0.7279 - val_balanced_accuracy: 0.8315 - val_specificity: 0.8589 - val_miss_rate: 0.1959 - val_fall_out: 0.1411 - val_mcc: 0.6261
Epoch 10/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3336 - accuracy: 0.8447 - recall: 0.7785 - precision: 0.6720 - AUROC: 0.9039 - AUPRC: 0.7093 - f1_score: 0.7213 - balanced_accuracy: 0.8231 - specificity: 0.8678 - miss_rate: 0.2215 - fall_out: 0.1322 - mcc: 0.6177 - val_loss: 0.3404 - val_accuracy: 0.8444 - val_recall: 0.7223 - val_precision: 0.6896 - val_AUROC: 0.9074 - val_AUPRC: 0.7209 - val_f1_score: 0.7056 - val_balanced_accuracy: 0.8046 - val_specificity: 0.8868 - val_miss_rate: 0.2777 - val_fall_out: 0.1132 - val_mcc: 0.6002
Epoch 11/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3305 - accuracy: 0.8468 - recall: 0.7717 - precision: 0.6787 - AUROC: 0.9056 - AUPRC: 0.7146 - f1_score: 0.7222 - balanced_accuracy: 0.8223 - specificity: 0.8729 - miss_rate: 0.2283 - fall_out: 0.1271 - mcc: 0.6195 - val_loss: 0.3424 - val_accuracy: 0.8462 - val_recall: 0.7776 - val_precision: 0.6755 - val_AUROC: 0.9081 - val_AUPRC: 0.7258 - val_f1_score: 0.7230 - val_balanced_accuracy: 0.8238 - val_specificity: 0.8700 - val_miss_rate: 0.2224 - val_fall_out: 0.1300 - val_mcc: 0.6202
Epoch 12/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3303 - accuracy: 0.8472 - recall: 0.7756 - precision: 0.6784 - AUROC: 0.9062 - AUPRC: 0.7161 - f1_score: 0.7237 - balanced_accuracy: 0.8238 - specificity: 0.8721 - miss_rate: 0.2244 - fall_out: 0.1279 - mcc: 0.6214 - val_loss: 0.3361 - val_accuracy: 0.8466 - val_recall: 0.7851 - val_precision: 0.6743 - val_AUROC: 0.9087 - val_AUPRC: 0.7203 - val_f1_score: 0.7255 - val_balanced_accuracy: 0.8266 - val_specificity: 0.8680 - val_miss_rate: 0.2149 - val_fall_out: 0.1320 - val_mcc: 0.6234
Epoch 13/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3278 - accuracy: 0.8494 - recall: 0.7776 - precision: 0.6829 - AUROC: 0.9078 - AUPRC: 0.7171 - f1_score: 0.7272 - balanced_accuracy: 0.8260 - specificity: 0.8744 - miss_rate: 0.2224 - fall_out: 0.1256 - mcc: 0.6263 - val_loss: 0.3355 - val_accuracy: 0.8474 - val_recall: 0.7950 - val_precision: 0.6730 - val_AUROC: 0.9098 - val_AUPRC: 0.7243 - val_f1_score: 0.7290 - val_balanced_accuracy: 0.8303 - val_specificity: 0.8656 - val_miss_rate: 0.2050 - val_fall_out: 0.1344 - val_mcc: 0.6279
Epoch 14/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3258 - accuracy: 0.8491 - recall: 0.7807 - precision: 0.6813 - AUROC: 0.9087 - AUPRC: 0.7210 - f1_score: 0.7276 - balanced_accuracy: 0.8268 - specificity: 0.8730 - miss_rate: 0.2193 - fall_out: 0.1270 - mcc: 0.6267 - val_loss: 0.3328 - val_accuracy: 0.8482 - val_recall: 0.7714 - val_precision: 0.6821 - val_AUROC: 0.9100 - val_AUPRC: 0.7245 - val_f1_score: 0.7240 - val_balanced_accuracy: 0.8231 - val_specificity: 0.8749 - val_miss_rate: 0.2286 - val_fall_out: 0.1251 - val_mcc: 0.6221
Epoch 15/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3245 - accuracy: 0.8503 - recall: 0.7700 - precision: 0.6875 - AUROC: 0.9097 - AUPRC: 0.7247 - f1_score: 0.7264 - balanced_accuracy: 0.8241 - specificity: 0.8782 - miss_rate: 0.2300 - fall_out: 0.1218 - mcc: 0.6258 - val_loss: 0.3372 - val_accuracy: 0.8478 - val_recall: 0.7644 - val_precision: 0.6835 - val_AUROC: 0.9105 - val_AUPRC: 0.7285 - val_f1_score: 0.7217 - val_balanced_accuracy: 0.8206 - val_specificity: 0.8769 - val_miss_rate: 0.2356 - val_fall_out: 0.1231 - val_mcc: 0.6193
Epoch 16/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3224 - accuracy: 0.8514 - recall: 0.7824 - precision: 0.6859 - AUROC: 0.9102 - AUPRC: 0.7257 - f1_score: 0.7310 - balanced_accuracy: 0.8289 - specificity: 0.8754 - miss_rate: 0.2176 - fall_out: 0.1246 - mcc: 0.6315 - val_loss: 0.3334 - val_accuracy: 0.8461 - val_recall: 0.8001 - val_precision: 0.6688 - val_AUROC: 0.9094 - val_AUPRC: 0.7238 - val_f1_score: 0.7286 - val_balanced_accuracy: 0.8311 - val_specificity: 0.8621 - val_miss_rate: 0.1999 - val_fall_out: 0.1379 - val_mcc: 0.6273
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 8 / 100
Confirmed: 417
Tentative: 142
Rejected: 0
Iteration: 9 / 100
Confirmed: 417
Tentative: 142
Rejected: 0
Iteration: 10 / 100
Confirmed: 417
Tentative: 142
Rejected: 0
Iteration: 11 / 100
Confirmed: 417
Tentative: 142
Rejected: 0
Iteration: 12 / 100
Confirmed: 445
Tentative: 114
Rejected: 0
Iteration: 13 / 100
Confirmed: 445
Tentative: 114
Rejected: 0
Iteration: 14 / 100
Confirmed: 445
Tentative: 114
Rejected: 0
Iteration: 15 / 100
Confirmed: 445
Tentative: 114
Rejected: 0
Iteration: 16 / 100
Confirmed: 450
Tentative: 104
Rejected: 5
Iteration: 17 / 100
Confirmed: 450
Tentative: 104
Rejected: 5
Iteration: 18 / 100
Confirmed: 450
Tentative: 104
Rejected: 5
Iteration: 19 / 100
Confirmed: 457
Tentative: 97
Rejected: 5
Iteration: 20 / 100
Confirmed: 457
Tentative: 97
Rejected: 5
Iteration: 21 / 100
Confirmed: 457
Tentative: 95
Rejected: 7
Iteration: 22 / 100
Confirmed: 464
Tentative: 88
Rejected: 7
Iteration: 23 / 100
Confirmed: 464
Tentative: 88
Rejected: 7
Iteration: 24 / 100
Confirmed: 464
Tentative: 88
Rejected: 7
Iteration: 25 / 100
Confirmed: 464
Tentative: 88
Rejected: 7
Iteration: 26 / 100
Confirmed: 464
Tentative: 84
Rejected: 11
Iteration: 27 / 100
Confirmed: 464
Tentative: 84
Rejected: 11
Iteration: 28 / 100
Confirmed: 464
Tentative: 84
Rejected: 11
Iteration: 29 / 100
Confirmed: 467
Tentative: 81
Rejected: 11
Iteration: 30 / 100
Confirmed: 467
Tentative: 81
Rejected: 11
Iteration: 31 / 100
Confirmed: 467
Tentative: 81
Rejected: 11
Iteration: 32 / 100
Confirmed: 470
Tentative: 78
Rejected: 11
Iteration: 33 / 100
Confirmed: 470
Tentative: 78
Rejected: 11
Iteration: 34 / 100
Confirmed: 472
Tentative: 76
Rejected: 11
Iteration: 35 / 100
Confirmed: 472
Tentative: 76
Rejected: 11
Iteration: 36 / 100
Confirmed: 472
Tentative: 76
Rejected: 11
Iteration: 37 / 100
Confirmed: 474
Tentative: 74
Rejected: 11
Iteration: 38 / 100
Confirmed: 474
Tentative: 71
Rejected: 14
Iteration: 39 / 100
Confirmed: 474
Tentative: 71
Rejected: 14
Iteration: 40 / 100
Confirmed: 478
Tentative: 67
Rejected: 14
Iteration: 41 / 100
Confirmed: 478
Tentative: 67
Rejected: 14
Iteration: 42 / 100
Confirmed: 478
Tentative: 61
Rejected: 20
Iteration: 43 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 44 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 45 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 46 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 47 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 48 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 49 / 100
Confirmed: 480
Tentative: 59
Rejected: 20
Iteration: 50 / 100
Confirmed: 480
Tentative: 59
Rejected: 20
Iteration: 51 / 100
Confirmed: 480
Tentative: 59
Rejected: 20
Iteration: 52 / 100
Confirmed: 480
Tentative: 59
Rejected: 20
Iteration: 53 / 100
Confirmed: 480
Tentative: 59
Rejected: 20
Iteration: 54 / 100
Confirmed: 480
Tentative: 59
Rejected: 20
Iteration: 55 / 100
Confirmed: 480
Tentative: 59
Rejected: 20
Iteration: 56 / 100
Confirmed: 480
Tentative: 59
Rejected: 20
Iteration: 57 / 100
Confirmed: 481
Tentative: 58
Rejected: 20
Iteration: 58 / 100
Confirmed: 481
Tentative: 58
Rejected: 20
Iteration: 59 / 100
Confirmed: 482
Tentative: 57
Rejected: 20
Iteration: 60 / 100
Confirmed: 482
Tentative: 57
Rejected: 20
Iteration: 61 / 100
Confirmed: 482
Tentative: 57
Rejected: 20
Iteration: 62 / 100
Confirmed: 482
Tentative: 56
Rejected: 21
Iteration: 63 / 100
Confirmed: 482
Tentative: 56
Rejected: 21
Iteration: 64 / 100
Confirmed: 482
Tentative: 56
Rejected: 21
Iteration: 65 / 100
Confirmed: 483
Tentative: 55
Rejected: 21
Iteration: 66 / 100
Confirmed: 483
Tentative: 53
Rejected: 23
Iteration: 67 / 100
Confirmed: 483
Tentative: 53
Rejected: 23
Iteration: 68 / 100
Confirmed: 483
Tentative: 53
Rejected: 23
Iteration: 69 / 100
Confirmed: 483
Tentative: 53
Rejected: 23
Iteration: 70 / 100
Confirmed: 483
Tentative: 53
Rejected: 23
Iteration: 71 / 100
Confirmed: 483
Tentative: 53
Rejected: 23
Iteration: 72 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 73 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 74 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 75 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 76 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 77 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 78 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 79 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 80 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 81 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 82 / 100
Confirmed: 485
Tentative: 51
Rejected: 23
Iteration: 83 / 100
Confirmed: 485
Tentative: 46
Rejected: 28
Iteration: 84 / 100
Confirmed: 485
Tentative: 46
Rejected: 28
Iteration: 85 / 100
Confirmed: 487
Tentative: 44
Rejected: 28
Iteration: 86 / 100
Confirmed: 487
Tentative: 44
Rejected: 28
Iteration: 87 / 100
Confirmed: 487
Tentative: 44
Rejected: 28
Iteration: 88 / 100
Confirmed: 487
Tentative: 44
Rejected: 28
Iteration: 89 / 100
Confirmed: 487
Tentative: 44
Rejected: 28
Iteration: 90 / 100
Confirmed: 488
Tentative: 42
Rejected: 29
Iteration: 91 / 100
Confirmed: 488
Tentative: 42
Rejected: 29
Iteration: 92 / 100
Confirmed: 488
Tentative: 39
Rejected: 32
Iteration: 93 / 100
Confirmed: 489
Tentative: 38
Rejected: 32
Iteration: 94 / 100
Confirmed: 489
Tentative: 38
Rejected: 32
Iteration: 95 / 100
Confirmed: 489
Tentative: 38
Rejected: 32
Iteration: 96 / 100
Confirmed: 489
Tentative: 38
Rejected: 32
Iteration: 97 / 100
Confirmed: 489
Tentative: 38
Rejected: 32
Iteration: 98 / 100
Confirmed: 489
Tentative: 38
Rejected: 32
Iteration: 99 / 100
Confirmed: 489
Tentative: 38
Rejected: 32
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 489
Tentative: 18
Rejected: 32
-- 489 Kept (correlated) features [Boruta]
-- 70 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_51 (Dense) (None, 72) 35280
dropout_27 (Dropout) (None, 72) 0
dense_52 (Dense) (None, 104) 7592
dropout_28 (Dropout) (None, 104) 0
dense_53 (Dense) (None, 40) 4200
dropout_29 (Dropout) (None, 40) 0
dense_54 (Dense) (None, 40) 1640
dropout_30 (Dropout) (None, 40) 0
dense_55 (Dense) (None, 1) 41
=================================================================
Total params: 48,753
Trainable params: 48,753
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 10s 28ms/step - loss: 0.4494 - accuracy: 0.7748 - recall: 0.4158 - precision: 0.5906 - AUROC: 0.8236 - AUPRC: 0.5599 - f1_score: 0.4880 - balanced_accuracy: 0.6578 - specificity: 0.8997 - miss_rate: 0.5842 - fall_out: 0.1003 - mcc: 0.3581 - val_loss: 0.3621 - val_accuracy: 0.8359 - val_recall: 0.7270 - val_precision: 0.6672 - val_AUROC: 0.8945 - val_AUPRC: 0.6778 - val_f1_score: 0.6958 - val_balanced_accuracy: 0.8004 - val_specificity: 0.8738 - val_miss_rate: 0.2730 - val_fall_out: 0.1262 - val_mcc: 0.5848
Epoch 2/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3740 - accuracy: 0.8202 - recall: 0.6410 - precision: 0.6552 - AUROC: 0.8822 - AUPRC: 0.6586 - f1_score: 0.6480 - balanced_accuracy: 0.7618 - specificity: 0.8826 - miss_rate: 0.3590 - fall_out: 0.1174 - mcc: 0.5274 - val_loss: 0.3516 - val_accuracy: 0.8392 - val_recall: 0.7607 - val_precision: 0.6648 - val_AUROC: 0.8997 - val_AUPRC: 0.6933 - val_f1_score: 0.7095 - val_balanced_accuracy: 0.8136 - val_specificity: 0.8665 - val_miss_rate: 0.2393 - val_fall_out: 0.1335 - val_mcc: 0.6017
Epoch 3/1000
313/313 [==============================] - 8s 27ms/step - loss: 0.3592 - accuracy: 0.8302 - recall: 0.6937 - precision: 0.6637 - AUROC: 0.8896 - AUPRC: 0.6774 - f1_score: 0.6784 - balanced_accuracy: 0.7857 - specificity: 0.8777 - miss_rate: 0.3063 - fall_out: 0.1223 - mcc: 0.5634 - val_loss: 0.3513 - val_accuracy: 0.8427 - val_recall: 0.7708 - val_precision: 0.6698 - val_AUROC: 0.9028 - val_AUPRC: 0.7053 - val_f1_score: 0.7167 - val_balanced_accuracy: 0.8193 - val_specificity: 0.8677 - val_miss_rate: 0.2292 - val_fall_out: 0.1323 - val_mcc: 0.6115
Epoch 4/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3528 - accuracy: 0.8345 - recall: 0.7235 - precision: 0.6649 - AUROC: 0.8929 - AUPRC: 0.6840 - f1_score: 0.6930 - balanced_accuracy: 0.7983 - specificity: 0.8731 - miss_rate: 0.2765 - fall_out: 0.1269 - mcc: 0.5810 - val_loss: 0.3483 - val_accuracy: 0.8428 - val_recall: 0.7978 - val_precision: 0.6623 - val_AUROC: 0.9042 - val_AUPRC: 0.7067 - val_f1_score: 0.7237 - val_balanced_accuracy: 0.8281 - val_specificity: 0.8584 - val_miss_rate: 0.2022 - val_fall_out: 0.1416 - val_mcc: 0.6204
Epoch 5/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3458 - accuracy: 0.8391 - recall: 0.7415 - precision: 0.6702 - AUROC: 0.8976 - AUPRC: 0.6977 - f1_score: 0.7040 - balanced_accuracy: 0.8073 - specificity: 0.8731 - miss_rate: 0.2585 - fall_out: 0.1269 - mcc: 0.5954 - val_loss: 0.3486 - val_accuracy: 0.8436 - val_recall: 0.7805 - val_precision: 0.6688 - val_AUROC: 0.9048 - val_AUPRC: 0.7083 - val_f1_score: 0.7204 - val_balanced_accuracy: 0.8230 - val_specificity: 0.8655 - val_miss_rate: 0.2195 - val_fall_out: 0.1345 - val_mcc: 0.6162
Epoch 6/1000
313/313 [==============================] - 8s 26ms/step - loss: 0.3426 - accuracy: 0.8405 - recall: 0.7492 - precision: 0.6712 - AUROC: 0.8992 - AUPRC: 0.6979 - f1_score: 0.7081 - balanced_accuracy: 0.8108 - specificity: 0.8723 - miss_rate: 0.2508 - fall_out: 0.1277 - mcc: 0.6006 - val_loss: 0.3491 - val_accuracy: 0.8409 - val_recall: 0.8175 - val_precision: 0.6532 - val_AUROC: 0.9050 - val_AUPRC: 0.7099 - val_f1_score: 0.7262 - val_balanced_accuracy: 0.8333 - val_specificity: 0.8490 - val_miss_rate: 0.1825 - val_fall_out: 0.1510 - val_mcc: 0.6237
-- HOLDOUT 9 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_8\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_56 (Dense) (None, 8) 4504
dropout_31 (Dropout) (None, 8) 0
dense_57 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_8\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 72, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.5, 'units_3': 8, 'dropout_3': 0.3, 'units_4': 8, 'dropout_4': 0.5}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_58 (Dense) (None, 136) 76160
dropout_32 (Dropout) (None, 136) 0
dense_59 (Dense) (None, 1) 137
=================================================================
Total params: 76,297
Trainable params: 76,297
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 8s 22ms/step - loss: 0.4139 - accuracy: 0.8173 - recall: 0.6855 - precision: 0.6353 - AUROC: 0.8675 - AUPRC: 0.6338 - f1_score: 0.6595 - balanced_accuracy: 0.7743 - specificity: 0.8631 - miss_rate: 0.3145 - fall_out: 0.1369 - mcc: 0.5356 - val_loss: 0.3485 - val_accuracy: 0.8361 - val_recall: 0.7539 - val_precision: 0.6598 - val_AUROC: 0.8975 - val_AUPRC: 0.6964 - val_f1_score: 0.7037 - val_balanced_accuracy: 0.8093 - val_specificity: 0.8647 - val_miss_rate: 0.2461 - val_fall_out: 0.1353 - val_mcc: 0.5936
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3468 - accuracy: 0.8390 - recall: 0.7326 - precision: 0.6728 - AUROC: 0.8978 - AUPRC: 0.7022 - f1_score: 0.7014 - balanced_accuracy: 0.8043 - specificity: 0.8761 - miss_rate: 0.2674 - fall_out: 0.1239 - mcc: 0.5925 - val_loss: 0.3376 - val_accuracy: 0.8432 - val_recall: 0.6874 - val_precision: 0.6999 - val_AUROC: 0.9035 - val_AUPRC: 0.7158 - val_f1_score: 0.6936 - val_balanced_accuracy: 0.7924 - val_specificity: 0.8974 - val_miss_rate: 0.3126 - val_fall_out: 0.1026 - val_mcc: 0.5883
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3361 - accuracy: 0.8444 - recall: 0.7376 - precision: 0.6843 - AUROC: 0.9041 - AUPRC: 0.7185 - f1_score: 0.7099 - balanced_accuracy: 0.8096 - specificity: 0.8816 - miss_rate: 0.2624 - fall_out: 0.1184 - mcc: 0.6047 - val_loss: 0.3506 - val_accuracy: 0.8372 - val_recall: 0.7124 - val_precision: 0.6750 - val_AUROC: 0.8954 - val_AUPRC: 0.6985 - val_f1_score: 0.6932 - val_balanced_accuracy: 0.7965 - val_specificity: 0.8806 - val_miss_rate: 0.2876 - val_fall_out: 0.1194 - val_mcc: 0.5829
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3286 - accuracy: 0.8470 - recall: 0.7386 - precision: 0.6903 - AUROC: 0.9080 - AUPRC: 0.7308 - f1_score: 0.7136 - balanced_accuracy: 0.8116 - specificity: 0.8847 - miss_rate: 0.2614 - fall_out: 0.1153 - mcc: 0.6100 - val_loss: 0.3304 - val_accuracy: 0.8443 - val_recall: 0.6725 - val_precision: 0.7093 - val_AUROC: 0.9075 - val_AUPRC: 0.7267 - val_f1_score: 0.6904 - val_balanced_accuracy: 0.7883 - val_specificity: 0.9041 - val_miss_rate: 0.3275 - val_fall_out: 0.0959 - val_mcc: 0.5869
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3239 - accuracy: 0.8497 - recall: 0.7398 - precision: 0.6968 - AUROC: 0.9108 - AUPRC: 0.7348 - f1_score: 0.7177 - balanced_accuracy: 0.8139 - specificity: 0.8880 - miss_rate: 0.2602 - fall_out: 0.1120 - mcc: 0.6159 - val_loss: 0.3274 - val_accuracy: 0.8471 - val_recall: 0.7080 - val_precision: 0.7023 - val_AUROC: 0.9096 - val_AUPRC: 0.7341 - val_f1_score: 0.7051 - val_balanced_accuracy: 0.8018 - val_specificity: 0.8955 - val_miss_rate: 0.2920 - val_fall_out: 0.1045 - val_mcc: 0.6019
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3202 - accuracy: 0.8516 - recall: 0.7466 - precision: 0.6990 - AUROC: 0.9129 - AUPRC: 0.7428 - f1_score: 0.7220 - balanced_accuracy: 0.8174 - specificity: 0.8881 - miss_rate: 0.2534 - fall_out: 0.1119 - mcc: 0.6216 - val_loss: 0.3257 - val_accuracy: 0.8498 - val_recall: 0.7694 - val_precision: 0.6865 - val_AUROC: 0.9098 - val_AUPRC: 0.7325 - val_f1_score: 0.7256 - val_balanced_accuracy: 0.8236 - val_specificity: 0.8777 - val_miss_rate: 0.2306 - val_fall_out: 0.1223 - val_mcc: 0.6246
Epoch 7/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3173 - accuracy: 0.8531 - recall: 0.7469 - precision: 0.7027 - AUROC: 0.9148 - AUPRC: 0.7464 - f1_score: 0.7241 - balanced_accuracy: 0.8185 - specificity: 0.8901 - miss_rate: 0.2531 - fall_out: 0.1099 - mcc: 0.6247 - val_loss: 0.3278 - val_accuracy: 0.8479 - val_recall: 0.7380 - val_precision: 0.6928 - val_AUROC: 0.9091 - val_AUPRC: 0.7371 - val_f1_score: 0.7147 - val_balanced_accuracy: 0.8121 - val_specificity: 0.8861 - val_miss_rate: 0.2620 - val_fall_out: 0.1139 - val_mcc: 0.6117
Epoch 8/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3157 - accuracy: 0.8530 - recall: 0.7460 - precision: 0.7027 - AUROC: 0.9157 - AUPRC: 0.7494 - f1_score: 0.7237 - balanced_accuracy: 0.8181 - specificity: 0.8902 - miss_rate: 0.2540 - fall_out: 0.1098 - mcc: 0.6242 - val_loss: 0.3273 - val_accuracy: 0.8496 - val_recall: 0.7789 - val_precision: 0.6829 - val_AUROC: 0.9103 - val_AUPRC: 0.7365 - val_f1_score: 0.7278 - val_balanced_accuracy: 0.8265 - val_specificity: 0.8742 - val_miss_rate: 0.2211 - val_fall_out: 0.1258 - val_mcc: 0.6271
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 8 / 100
Confirmed: 406
Tentative: 153
Rejected: 0
Iteration: 9 / 100
Confirmed: 406
Tentative: 153
Rejected: 0
Iteration: 10 / 100
Confirmed: 406
Tentative: 153
Rejected: 0
Iteration: 11 / 100
Confirmed: 406
Tentative: 153
Rejected: 0
Iteration: 12 / 100
Confirmed: 429
Tentative: 127
Rejected: 3
Iteration: 13 / 100
Confirmed: 429
Tentative: 127
Rejected: 3
Iteration: 14 / 100
Confirmed: 429
Tentative: 127
Rejected: 3
Iteration: 15 / 100
Confirmed: 429
Tentative: 127
Rejected: 3
Iteration: 16 / 100
Confirmed: 443
Tentative: 113
Rejected: 3
Iteration: 17 / 100
Confirmed: 443
Tentative: 113
Rejected: 3
Iteration: 18 / 100
Confirmed: 443
Tentative: 113
Rejected: 3
Iteration: 19 / 100
Confirmed: 454
Tentative: 102
Rejected: 3
Iteration: 20 / 100
Confirmed: 454
Tentative: 102
Rejected: 3
Iteration: 21 / 100
Confirmed: 454
Tentative: 102
Rejected: 3
Iteration: 22 / 100
Confirmed: 462
Tentative: 89
Rejected: 8
Iteration: 23 / 100
Confirmed: 462
Tentative: 89
Rejected: 8
Iteration: 24 / 100
Confirmed: 462
Tentative: 89
Rejected: 8
Iteration: 25 / 100
Confirmed: 462
Tentative: 89
Rejected: 8
Iteration: 26 / 100
Confirmed: 464
Tentative: 87
Rejected: 8
Iteration: 27 / 100
Confirmed: 464
Tentative: 87
Rejected: 8
Iteration: 28 / 100
Confirmed: 464
Tentative: 87
Rejected: 8
Iteration: 29 / 100
Confirmed: 467
Tentative: 84
Rejected: 8
Iteration: 30 / 100
Confirmed: 467
Tentative: 84
Rejected: 8
Iteration: 31 / 100
Confirmed: 467
Tentative: 84
Rejected: 8
Iteration: 32 / 100
Confirmed: 472
Tentative: 78
Rejected: 9
Iteration: 33 / 100
Confirmed: 472
Tentative: 78
Rejected: 9
Iteration: 34 / 100
Confirmed: 473
Tentative: 77
Rejected: 9
Iteration: 35 / 100
Confirmed: 473
Tentative: 77
Rejected: 9
Iteration: 36 / 100
Confirmed: 473
Tentative: 77
Rejected: 9
Iteration: 37 / 100
Confirmed: 473
Tentative: 77
Rejected: 9
Iteration: 38 / 100
Confirmed: 473
Tentative: 76
Rejected: 10
Iteration: 39 / 100
Confirmed: 473
Tentative: 76
Rejected: 10
Iteration: 40 / 100
Confirmed: 475
Tentative: 74
Rejected: 10
Iteration: 41 / 100
Confirmed: 475
Tentative: 74
Rejected: 10
Iteration: 42 / 100
Confirmed: 475
Tentative: 72
Rejected: 12
Iteration: 43 / 100
Confirmed: 475
Tentative: 72
Rejected: 12
Iteration: 44 / 100
Confirmed: 475
Tentative: 72
Rejected: 12
Iteration: 45 / 100
Confirmed: 475
Tentative: 72
Rejected: 12
Iteration: 46 / 100
Confirmed: 475
Tentative: 71
Rejected: 13
Iteration: 47 / 100
Confirmed: 475
Tentative: 71
Rejected: 13
Iteration: 48 / 100
Confirmed: 475
Tentative: 71
Rejected: 13
Iteration: 49 / 100
Confirmed: 475
Tentative: 68
Rejected: 16
Iteration: 50 / 100
Confirmed: 475
Tentative: 68
Rejected: 16
Iteration: 51 / 100
Confirmed: 477
Tentative: 66
Rejected: 16
Iteration: 52 / 100
Confirmed: 477
Tentative: 66
Rejected: 16
Iteration: 53 / 100
Confirmed: 477
Tentative: 66
Rejected: 16
Iteration: 54 / 100
Confirmed: 477
Tentative: 66
Rejected: 16
Iteration: 55 / 100
Confirmed: 477
Tentative: 66
Rejected: 16
Iteration: 56 / 100
Confirmed: 477
Tentative: 66
Rejected: 16
Iteration: 57 / 100
Confirmed: 478
Tentative: 64
Rejected: 17
Iteration: 58 / 100
Confirmed: 478
Tentative: 64
Rejected: 17
Iteration: 59 / 100
Confirmed: 479
Tentative: 63
Rejected: 17
Iteration: 60 / 100
Confirmed: 479
Tentative: 63
Rejected: 17
Iteration: 61 / 100
Confirmed: 479
Tentative: 63
Rejected: 17
Iteration: 62 / 100
Confirmed: 480
Tentative: 61
Rejected: 18
Iteration: 63 / 100
Confirmed: 480
Tentative: 61
Rejected: 18
Iteration: 64 / 100
Confirmed: 480
Tentative: 61
Rejected: 18
Iteration: 65 / 100
Confirmed: 482
Tentative: 59
Rejected: 18
Iteration: 66 / 100
Confirmed: 482
Tentative: 59
Rejected: 18
Iteration: 67 / 100
Confirmed: 483
Tentative: 58
Rejected: 18
Iteration: 68 / 100
Confirmed: 483
Tentative: 58
Rejected: 18
Iteration: 69 / 100
Confirmed: 483
Tentative: 58
Rejected: 18
Iteration: 70 / 100
Confirmed: 485
Tentative: 56
Rejected: 18
Iteration: 71 / 100
Confirmed: 485
Tentative: 56
Rejected: 18
Iteration: 72 / 100
Confirmed: 486
Tentative: 55
Rejected: 18
Iteration: 73 / 100
Confirmed: 486
Tentative: 54
Rejected: 19
Iteration: 74 / 100
Confirmed: 486
Tentative: 54
Rejected: 19
Iteration: 75 / 100
Confirmed: 486
Tentative: 54
Rejected: 19
Iteration: 76 / 100
Confirmed: 486
Tentative: 54
Rejected: 19
Iteration: 77 / 100
Confirmed: 487
Tentative: 53
Rejected: 19
Iteration: 78 / 100
Confirmed: 487
Tentative: 53
Rejected: 19
Iteration: 79 / 100
Confirmed: 487
Tentative: 53
Rejected: 19
Iteration: 80 / 100
Confirmed: 487
Tentative: 53
Rejected: 19
Iteration: 81 / 100
Confirmed: 487
Tentative: 53
Rejected: 19
Iteration: 82 / 100
Confirmed: 487
Tentative: 53
Rejected: 19
Iteration: 83 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 84 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 85 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 86 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 87 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 88 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 89 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 90 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 91 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 92 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 93 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 94 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 95 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 96 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 97 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 98 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
Iteration: 99 / 100
Confirmed: 488
Tentative: 46
Rejected: 25
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 488
Tentative: 19
Rejected: 25
-- 488 Kept (correlated) features [Boruta]
-- 71 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_60 (Dense) (None, 136) 66504
dropout_33 (Dropout) (None, 136) 0
dense_61 (Dense) (None, 1) 137
=================================================================
Total params: 66,641
Trainable params: 66,641
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 8s 23ms/step - loss: 0.4114 - accuracy: 0.8152 - recall: 0.6837 - precision: 0.6311 - AUROC: 0.8675 - AUPRC: 0.6347 - f1_score: 0.6563 - balanced_accuracy: 0.7723 - specificity: 0.8609 - miss_rate: 0.3163 - fall_out: 0.1391 - mcc: 0.5310 - val_loss: 0.3472 - val_accuracy: 0.8355 - val_recall: 0.7405 - val_precision: 0.6621 - val_AUROC: 0.8963 - val_AUPRC: 0.6997 - val_f1_score: 0.6991 - val_balanced_accuracy: 0.8045 - val_specificity: 0.8685 - val_miss_rate: 0.2595 - val_fall_out: 0.1315 - val_mcc: 0.5881
Epoch 2/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3489 - accuracy: 0.8384 - recall: 0.7340 - precision: 0.6708 - AUROC: 0.8967 - AUPRC: 0.6949 - f1_score: 0.7010 - balanced_accuracy: 0.8043 - specificity: 0.8747 - miss_rate: 0.2660 - fall_out: 0.1253 - mcc: 0.5917 - val_loss: 0.3451 - val_accuracy: 0.8409 - val_recall: 0.7768 - val_precision: 0.6640 - val_AUROC: 0.9000 - val_AUPRC: 0.7039 - val_f1_score: 0.7160 - val_balanced_accuracy: 0.8200 - val_specificity: 0.8632 - val_miss_rate: 0.2232 - val_fall_out: 0.1368 - val_mcc: 0.6100
Epoch 3/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3353 - accuracy: 0.8435 - recall: 0.7392 - precision: 0.6814 - AUROC: 0.9039 - AUPRC: 0.7150 - f1_score: 0.7091 - balanced_accuracy: 0.8095 - specificity: 0.8797 - miss_rate: 0.2608 - fall_out: 0.1203 - mcc: 0.6032 - val_loss: 0.3315 - val_accuracy: 0.8440 - val_recall: 0.7159 - val_precision: 0.6910 - val_AUROC: 0.9060 - val_AUPRC: 0.7211 - val_f1_score: 0.7032 - val_balanced_accuracy: 0.8023 - val_specificity: 0.8886 - val_miss_rate: 0.2841 - val_fall_out: 0.1114 - val_mcc: 0.5977
Epoch 4/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3302 - accuracy: 0.8468 - recall: 0.7405 - precision: 0.6891 - AUROC: 0.9069 - AUPRC: 0.7251 - f1_score: 0.7138 - balanced_accuracy: 0.8121 - specificity: 0.8838 - miss_rate: 0.2595 - fall_out: 0.1162 - mcc: 0.6101 - val_loss: 0.3286 - val_accuracy: 0.8455 - val_recall: 0.7444 - val_precision: 0.6847 - val_AUROC: 0.9072 - val_AUPRC: 0.7322 - val_f1_score: 0.7133 - val_balanced_accuracy: 0.8126 - val_specificity: 0.8807 - val_miss_rate: 0.2556 - val_fall_out: 0.1193 - val_mcc: 0.6088
Epoch 5/1000
313/313 [==============================] - 7s 22ms/step - loss: 0.3251 - accuracy: 0.8494 - recall: 0.7426 - precision: 0.6949 - AUROC: 0.9099 - AUPRC: 0.7316 - f1_score: 0.7180 - balanced_accuracy: 0.8146 - specificity: 0.8866 - miss_rate: 0.2574 - fall_out: 0.1134 - mcc: 0.6160 - val_loss: 0.3261 - val_accuracy: 0.8473 - val_recall: 0.7739 - val_precision: 0.6792 - val_AUROC: 0.9092 - val_AUPRC: 0.7337 - val_f1_score: 0.7235 - val_balanced_accuracy: 0.8234 - val_specificity: 0.8728 - val_miss_rate: 0.2261 - val_fall_out: 0.1272 - val_mcc: 0.6211
Epoch 6/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3211 - accuracy: 0.8506 - recall: 0.7487 - precision: 0.6957 - AUROC: 0.9123 - AUPRC: 0.7395 - f1_score: 0.7212 - balanced_accuracy: 0.8174 - specificity: 0.8861 - miss_rate: 0.2513 - fall_out: 0.1139 - mcc: 0.6201 - val_loss: 0.3257 - val_accuracy: 0.8468 - val_recall: 0.7683 - val_precision: 0.6799 - val_AUROC: 0.9099 - val_AUPRC: 0.7376 - val_f1_score: 0.7214 - val_balanced_accuracy: 0.8212 - val_specificity: 0.8742 - val_miss_rate: 0.2317 - val_fall_out: 0.1258 - val_mcc: 0.6185
Epoch 7/1000
313/313 [==============================] - 7s 22ms/step - loss: 0.3180 - accuracy: 0.8532 - recall: 0.7502 - precision: 0.7018 - AUROC: 0.9141 - AUPRC: 0.7459 - f1_score: 0.7252 - balanced_accuracy: 0.8196 - specificity: 0.8891 - miss_rate: 0.2498 - fall_out: 0.1109 - mcc: 0.6259 - val_loss: 0.3458 - val_accuracy: 0.8417 - val_recall: 0.8338 - val_precision: 0.6510 - val_AUROC: 0.9069 - val_AUPRC: 0.7316 - val_f1_score: 0.7312 - val_balanced_accuracy: 0.8391 - val_specificity: 0.8445 - val_miss_rate: 0.1662 - val_fall_out: 0.1555 - val_mcc: 0.6310
Epoch 8/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3162 - accuracy: 0.8533 - recall: 0.7485 - precision: 0.7026 - AUROC: 0.9152 - AUPRC: 0.7488 - f1_score: 0.7248 - balanced_accuracy: 0.8191 - specificity: 0.8898 - miss_rate: 0.2515 - fall_out: 0.1102 - mcc: 0.6255 - val_loss: 0.3310 - val_accuracy: 0.8462 - val_recall: 0.8142 - val_precision: 0.6650 - val_AUROC: 0.9102 - val_AUPRC: 0.7328 - val_f1_score: 0.7321 - val_balanced_accuracy: 0.8358 - val_specificity: 0.8573 - val_miss_rate: 0.1858 - val_fall_out: 0.1427 - val_mcc: 0.6321
-- HOLDOUT 10 --
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_9\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_62 (Dense) (None, 8) 4504
dropout_34 (Dropout) (None, 8) 0
dense_63 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_9\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 104, 'dropout_1': 0.3, 'units_2': 104, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.5, 'units_4': 72, 'dropout_4': 0.3}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF737', 'ZNF207']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF737', 'ZNF207']
-- 1 Correlated features: [Pearson]
['SAP130']
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_64 (Dense) (None, 136) 76160
dropout_35 (Dropout) (None, 136) 0
dense_65 (Dense) (None, 1) 137
=================================================================
Total params: 76,297
Trainable params: 76,297
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 8s 23ms/step - loss: 0.3827 - accuracy: 0.8248 - recall: 0.6791 - precision: 0.6549 - AUROC: 0.8796 - AUPRC: 0.6593 - f1_score: 0.6668 - balanced_accuracy: 0.7773 - specificity: 0.8755 - miss_rate: 0.3209 - fall_out: 0.1245 - mcc: 0.5482 - val_loss: 0.3363 - val_accuracy: 0.8451 - val_recall: 0.7105 - val_precision: 0.6959 - val_AUROC: 0.9052 - val_AUPRC: 0.7084 - val_f1_score: 0.7031 - val_balanced_accuracy: 0.8012 - val_specificity: 0.8920 - val_miss_rate: 0.2895 - val_fall_out: 0.1080 - val_mcc: 0.5984
Epoch 2/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3387 - accuracy: 0.8431 - recall: 0.7267 - precision: 0.6847 - AUROC: 0.9024 - AUPRC: 0.7133 - f1_score: 0.7051 - balanced_accuracy: 0.8051 - specificity: 0.8836 - miss_rate: 0.2733 - fall_out: 0.1164 - mcc: 0.5988 - val_loss: 0.3291 - val_accuracy: 0.8479 - val_recall: 0.7235 - val_precision: 0.6982 - val_AUROC: 0.9085 - val_AUPRC: 0.7177 - val_f1_score: 0.7106 - val_balanced_accuracy: 0.8073 - val_specificity: 0.8912 - val_miss_rate: 0.2765 - val_fall_out: 0.1088 - val_mcc: 0.6076
Epoch 3/1000
313/313 [==============================] - 7s 22ms/step - loss: 0.3284 - accuracy: 0.8475 - recall: 0.7335 - precision: 0.6933 - AUROC: 0.9080 - AUPRC: 0.7287 - f1_score: 0.7128 - balanced_accuracy: 0.8103 - specificity: 0.8871 - miss_rate: 0.2665 - fall_out: 0.1129 - mcc: 0.6095 - val_loss: 0.3283 - val_accuracy: 0.8471 - val_recall: 0.6601 - val_precision: 0.7235 - val_AUROC: 0.9108 - val_AUPRC: 0.7248 - val_f1_score: 0.6903 - val_balanced_accuracy: 0.7861 - val_specificity: 0.9122 - val_miss_rate: 0.3399 - val_fall_out: 0.0878 - val_mcc: 0.5902
Epoch 4/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3205 - accuracy: 0.8505 - recall: 0.7382 - precision: 0.6994 - AUROC: 0.9124 - AUPRC: 0.7432 - f1_score: 0.7183 - balanced_accuracy: 0.8139 - specificity: 0.8896 - miss_rate: 0.2618 - fall_out: 0.1104 - mcc: 0.6171 - val_loss: 0.3255 - val_accuracy: 0.8511 - val_recall: 0.7851 - val_precision: 0.6844 - val_AUROC: 0.9122 - val_AUPRC: 0.7296 - val_f1_score: 0.7313 - val_balanced_accuracy: 0.8296 - val_specificity: 0.8740 - val_miss_rate: 0.2149 - val_fall_out: 0.1260 - val_mcc: 0.6318
Epoch 5/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3155 - accuracy: 0.8546 - recall: 0.7456 - precision: 0.7070 - AUROC: 0.9156 - AUPRC: 0.7522 - f1_score: 0.7258 - balanced_accuracy: 0.8191 - specificity: 0.8925 - miss_rate: 0.2544 - fall_out: 0.1075 - mcc: 0.6274 - val_loss: 0.3215 - val_accuracy: 0.8519 - val_recall: 0.7245 - val_precision: 0.7084 - val_AUROC: 0.9131 - val_AUPRC: 0.7302 - val_f1_score: 0.7163 - val_balanced_accuracy: 0.8103 - val_specificity: 0.8962 - val_miss_rate: 0.2755 - val_fall_out: 0.1038 - val_mcc: 0.6162
Epoch 6/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3102 - accuracy: 0.8558 - recall: 0.7468 - precision: 0.7098 - AUROC: 0.9187 - AUPRC: 0.7626 - f1_score: 0.7278 - balanced_accuracy: 0.8203 - specificity: 0.8937 - miss_rate: 0.2532 - fall_out: 0.1063 - mcc: 0.6302 - val_loss: 0.3211 - val_accuracy: 0.8521 - val_recall: 0.7774 - val_precision: 0.6894 - val_AUROC: 0.9148 - val_AUPRC: 0.7403 - val_f1_score: 0.7308 - val_balanced_accuracy: 0.8278 - val_specificity: 0.8781 - val_miss_rate: 0.2226 - val_fall_out: 0.1219 - val_mcc: 0.6315
Epoch 7/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.3057 - accuracy: 0.8599 - recall: 0.7556 - precision: 0.7169 - AUROC: 0.9213 - AUPRC: 0.7683 - f1_score: 0.7357 - balanced_accuracy: 0.8259 - specificity: 0.8962 - miss_rate: 0.2444 - fall_out: 0.1038 - mcc: 0.6409 - val_loss: 0.3265 - val_accuracy: 0.8480 - val_recall: 0.6605 - val_precision: 0.7261 - val_AUROC: 0.9112 - val_AUPRC: 0.7383 - val_f1_score: 0.6917 - val_balanced_accuracy: 0.7869 - val_specificity: 0.9133 - val_miss_rate: 0.3395 - val_fall_out: 0.0867 - val_mcc: 0.5923
Epoch 8/1000
313/313 [==============================] - 7s 22ms/step - loss: 0.3026 - accuracy: 0.8608 - recall: 0.7538 - precision: 0.7200 - AUROC: 0.9230 - AUPRC: 0.7746 - f1_score: 0.7365 - balanced_accuracy: 0.8259 - specificity: 0.8980 - miss_rate: 0.2462 - fall_out: 0.1020 - mcc: 0.6423 - val_loss: 0.3164 - val_accuracy: 0.8532 - val_recall: 0.7756 - val_precision: 0.6926 - val_AUROC: 0.9163 - val_AUPRC: 0.7440 - val_f1_score: 0.7318 - val_balanced_accuracy: 0.8279 - val_specificity: 0.8802 - val_miss_rate: 0.2244 - val_fall_out: 0.1198 - val_mcc: 0.6331
Epoch 9/1000
313/313 [==============================] - 7s 21ms/step - loss: 0.2999 - accuracy: 0.8612 - recall: 0.7571 - precision: 0.7196 - AUROC: 0.9244 - AUPRC: 0.7790 - f1_score: 0.7379 - balanced_accuracy: 0.8272 - specificity: 0.8973 - miss_rate: 0.2429 - fall_out: 0.1027 - mcc: 0.6439 - val_loss: 0.3173 - val_accuracy: 0.8531 - val_recall: 0.7369 - val_precision: 0.7067 - val_AUROC: 0.9153 - val_AUPRC: 0.7419 - val_f1_score: 0.7215 - val_balanced_accuracy: 0.8152 - val_specificity: 0.8936 - val_miss_rate: 0.2631 - val_fall_out: 0.1064 - val_mcc: 0.6221
Epoch 10/1000
313/313 [==============================] - 7s 22ms/step - loss: 0.2944 - accuracy: 0.8655 - recall: 0.7595 - precision: 0.7303 - AUROC: 0.9275 - AUPRC: 0.7897 - f1_score: 0.7446 - balanced_accuracy: 0.8310 - specificity: 0.9024 - miss_rate: 0.2405 - fall_out: 0.0976 - mcc: 0.6536 - val_loss: 0.3176 - val_accuracy: 0.8540 - val_recall: 0.7454 - val_precision: 0.7057 - val_AUROC: 0.9157 - val_AUPRC: 0.7449 - val_f1_score: 0.7250 - val_balanced_accuracy: 0.8186 - val_specificity: 0.8918 - val_miss_rate: 0.2546 - val_fall_out: 0.1082 - val_mcc: 0.6262
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 559
Rejected: 0
Iteration: 8 / 100
Confirmed: 404
Tentative: 155
Rejected: 0
Iteration: 9 / 100
Confirmed: 404
Tentative: 155
Rejected: 0
Iteration: 10 / 100
Confirmed: 404
Tentative: 155
Rejected: 0
Iteration: 11 / 100
Confirmed: 404
Tentative: 155
Rejected: 0
Iteration: 12 / 100
Confirmed: 425
Tentative: 129
Rejected: 5
Iteration: 13 / 100
Confirmed: 425
Tentative: 129
Rejected: 5
Iteration: 14 / 100
Confirmed: 425
Tentative: 129
Rejected: 5
Iteration: 15 / 100
Confirmed: 425
Tentative: 123
Rejected: 11
Iteration: 16 / 100
Confirmed: 439
Tentative: 109
Rejected: 11
Iteration: 17 / 100
Confirmed: 439
Tentative: 109
Rejected: 11
Iteration: 18 / 100
Confirmed: 439
Tentative: 109
Rejected: 11
Iteration: 19 / 100
Confirmed: 446
Tentative: 102
Rejected: 11
Iteration: 20 / 100
Confirmed: 446
Tentative: 102
Rejected: 11
Iteration: 21 / 100
Confirmed: 446
Tentative: 102
Rejected: 11
Iteration: 22 / 100
Confirmed: 453
Tentative: 95
Rejected: 11
Iteration: 23 / 100
Confirmed: 453
Tentative: 95
Rejected: 11
Iteration: 24 / 100
Confirmed: 453
Tentative: 95
Rejected: 11
Iteration: 25 / 100
Confirmed: 453
Tentative: 94
Rejected: 12
Iteration: 26 / 100
Confirmed: 455
Tentative: 92
Rejected: 12
Iteration: 27 / 100
Confirmed: 455
Tentative: 92
Rejected: 12
Iteration: 28 / 100
Confirmed: 455
Tentative: 92
Rejected: 12
Iteration: 29 / 100
Confirmed: 459
Tentative: 88
Rejected: 12
Iteration: 30 / 100
Confirmed: 459
Tentative: 88
Rejected: 12
Iteration: 31 / 100
Confirmed: 459
Tentative: 88
Rejected: 12
Iteration: 32 / 100
Confirmed: 465
Tentative: 79
Rejected: 15
Iteration: 33 / 100
Confirmed: 465
Tentative: 79
Rejected: 15
Iteration: 34 / 100
Confirmed: 468
Tentative: 76
Rejected: 15
Iteration: 35 / 100
Confirmed: 468
Tentative: 76
Rejected: 15
Iteration: 36 / 100
Confirmed: 468
Tentative: 76
Rejected: 15
Iteration: 37 / 100
Confirmed: 470
Tentative: 74
Rejected: 15
Iteration: 38 / 100
Confirmed: 470
Tentative: 74
Rejected: 15
Iteration: 39 / 100
Confirmed: 470
Tentative: 74
Rejected: 15
Iteration: 40 / 100
Confirmed: 470
Tentative: 74
Rejected: 15
Iteration: 41 / 100
Confirmed: 470
Tentative: 71
Rejected: 18
Iteration: 42 / 100
Confirmed: 470
Tentative: 71
Rejected: 18
Iteration: 43 / 100
Confirmed: 473
Tentative: 68
Rejected: 18
Iteration: 44 / 100
Confirmed: 473
Tentative: 68
Rejected: 18
Iteration: 45 / 100
Confirmed: 473
Tentative: 68
Rejected: 18
Iteration: 46 / 100
Confirmed: 475
Tentative: 66
Rejected: 18
Iteration: 47 / 100
Confirmed: 475
Tentative: 66
Rejected: 18
Iteration: 48 / 100
Confirmed: 475
Tentative: 66
Rejected: 18
Iteration: 49 / 100
Confirmed: 475
Tentative: 66
Rejected: 18
Iteration: 50 / 100
Confirmed: 475
Tentative: 66
Rejected: 18
Iteration: 51 / 100
Confirmed: 477
Tentative: 64
Rejected: 18
Iteration: 52 / 100
Confirmed: 477
Tentative: 64
Rejected: 18
Iteration: 53 / 100
Confirmed: 477
Tentative: 62
Rejected: 20
Iteration: 54 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 55 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 56 / 100
Confirmed: 479
Tentative: 60
Rejected: 20
Iteration: 57 / 100
Confirmed: 480
Tentative: 58
Rejected: 21
Iteration: 58 / 100
Confirmed: 480
Tentative: 58
Rejected: 21
Iteration: 59 / 100
Confirmed: 480
Tentative: 58
Rejected: 21
Iteration: 60 / 100
Confirmed: 480
Tentative: 58
Rejected: 21
Iteration: 61 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 62 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 63 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 64 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 65 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 66 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 67 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 68 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 69 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 70 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 71 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 72 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 73 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 74 / 100
Confirmed: 480
Tentative: 54
Rejected: 25
Iteration: 75 / 100
Confirmed: 480
Tentative: 53
Rejected: 26
Iteration: 76 / 100
Confirmed: 480
Tentative: 53
Rejected: 26
Iteration: 77 / 100
Confirmed: 480
Tentative: 53
Rejected: 26
Iteration: 78 / 100
Confirmed: 480
Tentative: 53
Rejected: 26
Iteration: 79 / 100
Confirmed: 480
Tentative: 53
Rejected: 26
Iteration: 80 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 81 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 82 / 100
Confirmed: 481
Tentative: 51
Rejected: 27
Iteration: 83 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 84 / 100
Confirmed: 482
Tentative: 49
Rejected: 28
Iteration: 85 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 86 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 87 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 88 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 89 / 100
Confirmed: 483
Tentative: 48
Rejected: 28
Iteration: 90 / 100
Confirmed: 483
Tentative: 47
Rejected: 29
Iteration: 91 / 100
Confirmed: 483
Tentative: 47
Rejected: 29
Iteration: 92 / 100
Confirmed: 483
Tentative: 47
Rejected: 29
Iteration: 93 / 100
Confirmed: 483
Tentative: 47
Rejected: 29
Iteration: 94 / 100
Confirmed: 483
Tentative: 47
Rejected: 29
Iteration: 95 / 100
Confirmed: 483
Tentative: 47
Rejected: 29
Iteration: 96 / 100
Confirmed: 483
Tentative: 47
Rejected: 29
Iteration: 97 / 100
Confirmed: 483
Tentative: 47
Rejected: 29
Iteration: 98 / 100
Confirmed: 484
Tentative: 45
Rejected: 30
Iteration: 99 / 100
Confirmed: 484
Tentative: 45
Rejected: 30
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 484
Tentative: 26
Rejected: 30
-- 484 Kept (correlated) features [Boruta]
-- 75 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_66 (Dense) (None, 136) 65960
dropout_36 (Dropout) (None, 136) 0
dense_67 (Dense) (None, 1) 137
=================================================================
Total params: 66,097
Trainable params: 66,097
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 7s 20ms/step - loss: 0.3852 - accuracy: 0.8244 - recall: 0.6798 - precision: 0.6538 - AUROC: 0.8784 - AUPRC: 0.6580 - f1_score: 0.6665 - balanced_accuracy: 0.7773 - specificity: 0.8747 - miss_rate: 0.3202 - fall_out: 0.1253 - mcc: 0.5476 - val_loss: 0.3354 - val_accuracy: 0.8458 - val_recall: 0.7322 - val_precision: 0.6896 - val_AUROC: 0.9057 - val_AUPRC: 0.7098 - val_f1_score: 0.7102 - val_balanced_accuracy: 0.8087 - val_specificity: 0.8853 - val_miss_rate: 0.2678 - val_fall_out: 0.1147 - val_mcc: 0.6058
Epoch 2/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3395 - accuracy: 0.8416 - recall: 0.7294 - precision: 0.6801 - AUROC: 0.9015 - AUPRC: 0.7106 - f1_score: 0.7039 - balanced_accuracy: 0.8050 - specificity: 0.8806 - miss_rate: 0.2706 - fall_out: 0.1194 - mcc: 0.5967 - val_loss: 0.3294 - val_accuracy: 0.8477 - val_recall: 0.7834 - val_precision: 0.6772 - val_AUROC: 0.9093 - val_AUPRC: 0.7211 - val_f1_score: 0.7264 - val_balanced_accuracy: 0.8267 - val_specificity: 0.8700 - val_miss_rate: 0.2166 - val_fall_out: 0.1300 - val_mcc: 0.6248
Epoch 3/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3280 - accuracy: 0.8465 - recall: 0.7357 - precision: 0.6902 - AUROC: 0.9084 - AUPRC: 0.7308 - f1_score: 0.7122 - balanced_accuracy: 0.8104 - specificity: 0.8851 - miss_rate: 0.2643 - fall_out: 0.1149 - mcc: 0.6083 - val_loss: 0.3217 - val_accuracy: 0.8488 - val_recall: 0.7115 - val_precision: 0.7053 - val_AUROC: 0.9121 - val_AUPRC: 0.7331 - val_f1_score: 0.7084 - val_balanced_accuracy: 0.8040 - val_specificity: 0.8966 - val_miss_rate: 0.2885 - val_fall_out: 0.1034 - val_mcc: 0.6063
Epoch 4/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3216 - accuracy: 0.8507 - recall: 0.7406 - precision: 0.6989 - AUROC: 0.9122 - AUPRC: 0.7426 - f1_score: 0.7192 - balanced_accuracy: 0.8148 - specificity: 0.8890 - miss_rate: 0.2594 - fall_out: 0.1110 - mcc: 0.6181 - val_loss: 0.3210 - val_accuracy: 0.8501 - val_recall: 0.7574 - val_precision: 0.6913 - val_AUROC: 0.9132 - val_AUPRC: 0.7339 - val_f1_score: 0.7229 - val_balanced_accuracy: 0.8199 - val_specificity: 0.8823 - val_miss_rate: 0.2426 - val_fall_out: 0.1177 - val_mcc: 0.6216
Epoch 5/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3165 - accuracy: 0.8536 - recall: 0.7437 - precision: 0.7051 - AUROC: 0.9149 - AUPRC: 0.7501 - f1_score: 0.7239 - balanced_accuracy: 0.8177 - specificity: 0.8918 - miss_rate: 0.2563 - fall_out: 0.1082 - mcc: 0.6248 - val_loss: 0.3196 - val_accuracy: 0.8507 - val_recall: 0.7673 - val_precision: 0.6894 - val_AUROC: 0.9141 - val_AUPRC: 0.7383 - val_f1_score: 0.7263 - val_balanced_accuracy: 0.8235 - val_specificity: 0.8797 - val_miss_rate: 0.2327 - val_fall_out: 0.1203 - val_mcc: 0.6257
Epoch 6/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3125 - accuracy: 0.8555 - recall: 0.7470 - precision: 0.7090 - AUROC: 0.9172 - AUPRC: 0.7571 - f1_score: 0.7275 - balanced_accuracy: 0.8201 - specificity: 0.8933 - miss_rate: 0.2530 - fall_out: 0.1067 - mcc: 0.6297 - val_loss: 0.3221 - val_accuracy: 0.8517 - val_recall: 0.7144 - val_precision: 0.7122 - val_AUROC: 0.9128 - val_AUPRC: 0.7409 - val_f1_score: 0.7133 - val_balanced_accuracy: 0.8069 - val_specificity: 0.8995 - val_miss_rate: 0.2856 - val_fall_out: 0.1005 - val_mcc: 0.6133
Epoch 7/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3080 - accuracy: 0.8575 - recall: 0.7474 - precision: 0.7139 - AUROC: 0.9197 - AUPRC: 0.7662 - f1_score: 0.7303 - balanced_accuracy: 0.8216 - specificity: 0.8958 - miss_rate: 0.2526 - fall_out: 0.1042 - mcc: 0.6338 - val_loss: 0.3173 - val_accuracy: 0.8534 - val_recall: 0.7592 - val_precision: 0.6990 - val_AUROC: 0.9151 - val_AUPRC: 0.7430 - val_f1_score: 0.7278 - val_balanced_accuracy: 0.8227 - val_specificity: 0.8862 - val_miss_rate: 0.2408 - val_fall_out: 0.1138 - val_mcc: 0.6288
Epoch 8/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3053 - accuracy: 0.8596 - recall: 0.7532 - precision: 0.7170 - AUROC: 0.9213 - AUPRC: 0.7690 - f1_score: 0.7347 - balanced_accuracy: 0.8249 - specificity: 0.8966 - miss_rate: 0.2468 - fall_out: 0.1034 - mcc: 0.6396 - val_loss: 0.3167 - val_accuracy: 0.8529 - val_recall: 0.7196 - val_precision: 0.7131 - val_AUROC: 0.9156 - val_AUPRC: 0.7447 - val_f1_score: 0.7163 - val_balanced_accuracy: 0.8094 - val_specificity: 0.8993 - val_miss_rate: 0.2804 - val_fall_out: 0.1007 - val_mcc: 0.6170
Epoch 9/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.3022 - accuracy: 0.8611 - recall: 0.7544 - precision: 0.7207 - AUROC: 0.9231 - AUPRC: 0.7742 - f1_score: 0.7371 - balanced_accuracy: 0.8263 - specificity: 0.8983 - miss_rate: 0.2456 - fall_out: 0.1017 - mcc: 0.6431 - val_loss: 0.3303 - val_accuracy: 0.8493 - val_recall: 0.8092 - val_precision: 0.6732 - val_AUROC: 0.9120 - val_AUPRC: 0.7336 - val_f1_score: 0.7349 - val_balanced_accuracy: 0.8362 - val_specificity: 0.8633 - val_miss_rate: 0.1908 - val_fall_out: 0.1367 - val_mcc: 0.6361
Epoch 10/1000
313/313 [==============================] - 6s 19ms/step - loss: 0.2979 - accuracy: 0.8634 - recall: 0.7608 - precision: 0.7241 - AUROC: 0.9256 - AUPRC: 0.7830 - f1_score: 0.7420 - balanced_accuracy: 0.8300 - specificity: 0.8991 - miss_rate: 0.2392 - fall_out: 0.1009 - mcc: 0.6496 - val_loss: 0.3176 - val_accuracy: 0.8541 - val_recall: 0.7413 - val_precision: 0.7074 - val_AUROC: 0.9165 - val_AUPRC: 0.7409 - val_f1_score: 0.7240 - val_balanced_accuracy: 0.8173 - val_specificity: 0.8933 - val_miss_rate: 0.2587 - val_fall_out: 0.1067 - val_mcc: 0.6252
MLP_metrics_estimate = model_metrics_holdout_estimate(MLP_metrics_promoters["boruta_False"], number_of_splits)
print("-- PROMOTERS -- withOUT BORUTA FEATURE SELECTION")
print(f"MLP Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {MLP_metrics_estimate['accuracy_train']} -- test - {MLP_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {MLP_metrics_estimate['AUROC_train']} -- test - {MLP_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {MLP_metrics_estimate['AUPRC_train']} -- test - {MLP_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("MLP - Train history:")
plot_train_history(MLP_history_promoters["boruta_False"])
print("-"*100)
MLP_metrics_estimate = model_metrics_holdout_estimate(MLP_metrics_promoters["boruta_True"], number_of_splits)
print("-- PROMOTERS -- with BORUTA FEATURE SELECTION (selective)")
print(f"MLP Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {MLP_metrics_estimate['accuracy_train']} -- test - {MLP_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {MLP_metrics_estimate['AUROC_train']} -- test - {MLP_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {MLP_metrics_estimate['AUPRC_train']} -- test - {MLP_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("MLP - Train history:")
plot_train_history(MLP_history_promoters["boruta_True"])
-- PROMOTERS -- withOUT BORUTA FEATURE SELECTION MLP Metrics - 10-holdouts estimate: Accuracy : train - 0.8585627753303966 -- test - 0.8457426039945937 AUROC : train - 0.9228826231988739 -- test - 0.908225762344436 AUPRC : train - 0.7754590575471727 -- test - 0.730489182316224 -------------------------------------------------------------------------------- MLP - Train history:
---------------------------------------------------------------------------------------------------- -- PROMOTERS -- with BORUTA FEATURE SELECTION (selective) MLP Metrics - 10-holdouts estimate: Accuracy : train - 0.8610082098518221 -- test - 0.8482404765480303 AUROC : train - 0.926005039875637 -- test - 0.9115797167326514 AUPRC : train - 0.7821367946295082 -- test - 0.7343833693474171 -------------------------------------------------------------------------------- MLP - Train history:
MLP_metrics_promoters
{'boruta_True': [{'accuracy_train': 0.8691805166199439,
'accuracy_test': 0.8477248836161586,
'AUROC_train': 0.9316258866265176,
'AUROC_test': 0.910291025104564,
'AUPRC_train': 0.799867754920471,
'AUPRC_test': 0.7356074096598941},
{'accuracy_train': 0.8662520024028835,
'accuracy_test': 0.8536817339940932,
'AUROC_train': 0.9302799546264787,
'AUROC_test': 0.9131811261346486,
'AUPRC_train': 0.7932045839765405,
'AUPRC_test': 0.7428782216184122},
{'accuracy_train': 0.8626977372847416,
'accuracy_test': 0.8506782800220253,
'AUROC_train': 0.9281695354326,
'AUROC_test': 0.9130397554573177,
'AUPRC_train': 0.7909347900957879,
'AUPRC_test': 0.7381452849862872},
{'accuracy_train': 0.8499574489387265,
'accuracy_test': 0.8477248836161586,
'AUROC_train': 0.9153795789504894,
'AUROC_test': 0.9125136699537361,
'AUPRC_train': 0.750664209218836,
'AUPRC_test': 0.7323172063854766},
{'accuracy_train': 0.8612710252302763,
'accuracy_test': 0.8457225809681134,
'AUROC_train': 0.9258704638108337,
'AUROC_test': 0.9105087434057766,
'AUPRC_train': 0.784950162534985,
'AUPRC_test': 0.7319939432947129},
{'accuracy_train': 0.8674784741690028,
'accuracy_test': 0.8483255744105721,
'AUROC_train': 0.9334358161482911,
'AUROC_test': 0.9139946764705652,
'AUPRC_train': 0.8052829865282306,
'AUPRC_test': 0.7451973756461822},
{'accuracy_train': 0.8627352823388066,
'accuracy_test': 0.8474245382189518,
'AUROC_train': 0.926746139237821,
'AUROC_test': 0.9100774741929331,
'AUPRC_train': 0.7885536108393832,
'AUPRC_test': 0.7337145239573722},
{'accuracy_train': 0.8432243692430917,
'accuracy_test': 0.8408669970466036,
'AUROC_train': 0.9109212425883355,
'AUROC_test': 0.9050789815187722,
'AUPRC_train': 0.728984395385126,
'AUPRC_test': 0.7100634088789423},
{'accuracy_train': 0.8554139967961554,
'accuracy_test': 0.8461730990639235,
'AUROC_train': 0.9222970002663318,
'AUROC_test': 0.9103970940013928,
'AUPRC_train': 0.7671168657455891,
'AUPRC_test': 0.7330813324250314},
{'accuracy_train': 0.8718712454945935,
'accuracy_test': 0.8540821945237023,
'AUROC_train': 0.9353247810686701,
'AUROC_test': 0.9167146210868081,
'AUPRC_train': 0.8118085870501328,
'AUPRC_test': 0.7408349866218598}],
'boruta_False': [{'accuracy_train': 0.8526356627953544,
'accuracy_test': 0.8393152124943686,
'AUROC_train': 0.9144219743616089,
'AUROC_test': 0.8963903994858343,
'AUPRC_train': 0.7558463324996817,
'AUPRC_test': 0.7149675850696401},
{'accuracy_train': 0.8560147176611934,
'accuracy_test': 0.8467237322921359,
'AUROC_train': 0.9244265245958123,
'AUROC_test': 0.912572922775458,
'AUPRC_train': 0.7769055278014887,
'AUPRC_test': 0.736933522298457},
{'accuracy_train': 0.8621470764917901,
'accuracy_test': 0.8482755168443711,
'AUROC_train': 0.9277621437813074,
'AUROC_test': 0.9135159042502662,
'AUPRC_train': 0.7899769296404695,
'AUPRC_test': 0.7389647802653102},
{'accuracy_train': 0.8622972567080497,
'accuracy_test': 0.8513790859488412,
'AUROC_train': 0.9265879915571308,
'AUROC_test': 0.9169710758302656,
'AUPRC_train': 0.7890925427547103,
'AUPRC_test': 0.7495599749307041},
{'accuracy_train': 0.8462029435322387,
'accuracy_test': 0.8338088802122441,
'AUROC_train': 0.9110510922412608,
'AUROC_test': 0.8955967963568772,
'AUPRC_train': 0.7483677198559271,
'AUPRC_test': 0.7059186479609867},
{'accuracy_train': 0.8565153183820585,
'accuracy_test': 0.8460729839315212,
'AUROC_train': 0.9202883367659968,
'AUROC_test': 0.9089702831757576,
'AUPRC_train': 0.7637606023502217,
'AUPRC_test': 0.732585631633638},
{'accuracy_train': 0.8597692230676812,
'accuracy_test': 0.8421184362016318,
'AUROC_train': 0.9220469465654707,
'AUROC_test': 0.9019340482140151,
'AUPRC_train': 0.7825692645722133,
'AUPRC_test': 0.7201491596335108},
{'accuracy_train': 0.8563901682018422,
'accuracy_test': 0.8461230414977223,
'AUROC_train': 0.9211968275341005,
'AUROC_test': 0.9094436580704608,
'AUPRC_train': 0.7525047798842479,
'AUPRC_test': 0.7239984541701325},
{'accuracy_train': 0.8601947336804165,
'accuracy_test': 0.8495770135656004,
'AUROC_train': 0.9242152700608882,
'AUROC_test': 0.9108330016431423,
'AUPRC_train': 0.7782057358980937,
'AUPRC_test': 0.7369504411988923},
{'accuracy_train': 0.87346065278334,
'accuracy_test': 0.8540321369575011,
'AUROC_train': 0.9368291245251653,
'AUROC_test': 0.9160295336422829,
'AUPRC_train': 0.8173611402146739,
'AUPRC_test': 0.744863626000968}]}
# Saving results
import json
# Saving metrics
file = open("results/MLP_metrics_promoters.txt", "w")
string = repr(MLP_metrics_promoters)
file.write("MLP_metrics_promoters = " + string + "\n")
file.close()
# Saving history
for boruta in ("boruta_True", "boruta_False"):
with open("results/MLP_history_promoters_" + boruta + ".json", 'w') as h:
for holdout in range(number_of_splits):
json.dump(MLP_history_promoters[boruta][holdout].history, h)
The Wilcoxon signed-rank test is a non-parametric statistical hypothesis test which checks the null hypothesis that two related paired samples come from the same distribution. Here, this test is used to determine whether there is any statistically significant difference in the results produced by two different model architectures.
from scipy.stats import wilcoxon
MLP_distinct_metrics_promoters = {}
MLP_distinct_metrics_promoters["boruta_True"] = {}
MLP_distinct_metrics_promoters["boruta_False"] = {}
for boruta in ("boruta_True", "boruta_False"):
MLP_distinct_metrics_promoters[boruta]["accuracy_test"] = []
MLP_distinct_metrics_promoters[boruta]["AUROC_test"] = []
MLP_distinct_metrics_promoters[boruta]["AUPRC_test"] = []
for holdout in MLP_metrics_promoters[boruta]:
MLP_distinct_metrics_promoters[boruta]["accuracy_test"].append(holdout['accuracy_test'])
MLP_distinct_metrics_promoters[boruta]["AUROC_test"].append(holdout['AUROC_test'])
MLP_distinct_metrics_promoters[boruta]["AUPRC_test"].append(holdout['AUPRC_test'])
print("MLP - Promoters - Wilcoxon test - Boruta model performances VS Non-Boruta model performances")
print(f"-Accuracy: {wilcoxon(MLP_distinct_metrics_promoters['boruta_True']['accuracy_test'], MLP_distinct_metrics_promoters['boruta_False']['accuracy_test'])}")
print(f"-AUROC: {wilcoxon(MLP_distinct_metrics_promoters['boruta_True']['AUROC_test'], MLP_distinct_metrics_promoters['boruta_False']['AUROC_test'])}")
print(f"-AUPRC: {wilcoxon(MLP_distinct_metrics_promoters['boruta_True']['AUPRC_test'], MLP_distinct_metrics_promoters['boruta_False']['AUPRC_test'])}")
MLP - Promoters - Wilcoxon test - Boruta model performances VS Non-Boruta model performances -Accuracy: WilcoxonResult(statistic=15.0, pvalue=0.232421875) -AUROC: WilcoxonResult(statistic=14.0, pvalue=0.193359375) -AUPRC: WilcoxonResult(statistic=21.0, pvalue=0.556640625)
print("---- ENHANCERS ----")
MLP_metrics_enhancers = {}
MLP_history_enhancers = {}
MLP_metrics_enhancers["boruta_True"] = []
MLP_metrics_enhancers["boruta_False"] = []
MLP_history_enhancers["boruta_True"] = []
MLP_history_enhancers["boruta_False"] = []
# Generate holdouts
for holdout_number, (train_indices, test_indices) in enumerate(holdouts_generator.split(enhancers_epigenomes, enhancers_labels)):
print(f"-- HOLDOUT {holdout_number+1} --")
# Train/Test data
x_train_enhancers, x_test_enhancers = enhancers_epigenomes.iloc[train_indices], enhancers_epigenomes.iloc[test_indices]
y_train_enhancers, y_test_enhancers = enhancers_labels.iloc[train_indices], enhancers_labels.iloc[test_indices]
## Hyperparameter tuning
# Generate holdouts
for holdout_number_tuning, (train_indices_tuning, val_indices_tuning) in enumerate(holdouts_generator_tuning.split(x_train_enhancers, y_train_enhancers)):
# Train/Validation data
x_train_enhancers_tuning, x_val_enhancers_tuning = x_train_enhancers.iloc[train_indices_tuning], x_train_enhancers.iloc[val_indices_tuning]
y_train_enhancers_tuning, y_val_enhancers_tuning = y_train_enhancers.iloc[train_indices_tuning], y_train_enhancers.iloc[val_indices_tuning]
hp = kt.HyperParameters()
best_hyperparameters = hyperparameter_tuning(
x_train_enhancers_tuning.values,
x_val_enhancers_tuning.values,
y_train_enhancers_tuning.values,
y_val_enhancers_tuning.values,
build_MLP_hypermodel,
name = "MLP_hypermodel_enhancers_" + str(holdout_number),
directory='MLP_hypermodel_enhancers',
max_trials = 8,
epochs = 50,
batch_size = 256
)
## Remove uncorrelated features with the output
uncorrelated_features = uncorrelated_features_test(x_train_enhancers, y_train_enhancers)
for feature in (x_train_enhancers.columns):
if feature in (uncorrelated_features):
x_train_enhancers = x_train_enhancers.drop(columns=feature)
x_test_enhancers = x_test_enhancers.drop(columns=feature)
## Remove correlated features with eachother
correlated_features = correlated_features_test(x_train_enhancers)
for feature in (x_train_enhancers.columns):
if feature in (correlated_features):
x_train_enhancers = x_train_enhancers.drop(columns=feature)
x_test_enhancers = x_test_enhancers.drop(columns=feature)
## Boruta feature selection
for use_Boruta in (False, True):
if use_Boruta:
print("-- Running Boruta Feature Selection:")
kept_features = boruta_feature_selection(x_train_enhancers, y_train_enhancers)
x_train_enhancers = x_train_enhancers[kept_features]
x_test_enhancers = x_test_enhancers[kept_features]
# Build MLP with best set of hyperparameters
MLP = build_MLP(best_hyperparameters, x_train_enhancers.shape)
print("- Training model:\n")
MLP_holdout_metrics, MLP_holdout_history = train_model(
MLP,
x_train_enhancers.values,
x_test_enhancers.values,
y_train_enhancers.values,
y_test_enhancers.values,
epochs,
batch_size
)
MLP_metrics_enhancers["boruta_" + str(use_Boruta)].append(MLP_holdout_metrics)
MLP_history_enhancers["boruta_" + str(use_Boruta)].append(MLP_holdout_history)
Trial 8 Complete [00h 00m 28s]
multi_objective: -2.3371983766555786
Best multi_objective So Far: -2.357012093067169
Total elapsed time: 00h 04m 54s
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 3, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 8, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.3}
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF282', 'SNRNP70']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF282', 'SNRNP70']
-- 0 Correlated features: [Pearson]
[]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_4 (Dense) (None, 136) 76296
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 8) 1096
dropout_4 (Dropout) (None, 8) 0
dense_6 (Dense) (None, 8) 72
dropout_5 (Dropout) (None, 8) 0
dense_7 (Dense) (None, 1) 9
=================================================================
Total params: 77,473
Trainable params: 77,473
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
198/198 [==============================] - 6s 26ms/step - loss: 0.4433 - accuracy: 0.8354 - recall: 0.0751 - precision: 0.1248 - AUROC: 0.6633 - AUPRC: 0.1613 - f1_score: 0.0937 - balanced_accuracy: 0.5039 - specificity: 0.9327 - miss_rate: 0.9249 - fall_out: 0.0673 - mcc: 0.0097 - val_loss: 0.2786 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8262 - val_AUPRC: 0.4310 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 2/1000
198/198 [==============================] - 5s 24ms/step - loss: 0.3268 - accuracy: 0.8838 - recall: 0.0045 - precision: 0.1354 - AUROC: 0.7764 - AUPRC: 0.2613 - f1_score: 0.0088 - balanced_accuracy: 0.5004 - specificity: 0.9963 - miss_rate: 0.9955 - fall_out: 0.0037 - mcc: 0.0043 - val_loss: 0.2759 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8364 - val_AUPRC: 0.4051 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 3/1000
198/198 [==============================] - 5s 23ms/step - loss: 0.3048 - accuracy: 0.8864 - recall: 0.0035 - precision: 0.3922 - AUROC: 0.8016 - AUPRC: 0.3049 - f1_score: 0.0069 - balanced_accuracy: 0.5014 - specificity: 0.9993 - miss_rate: 0.9965 - fall_out: 6.9064e-04 - mcc: 0.0279 - val_loss: 0.2748 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8397 - val_AUPRC: 0.4215 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 4/1000
198/198 [==============================] - 5s 23ms/step - loss: 0.2925 - accuracy: 0.8863 - recall: 0.0000e+00 - precision: 0.0000e+00 - AUROC: 0.8175 - AUPRC: 0.3298 - f1_score: 0.0000e+00 - balanced_accuracy: 0.4998 - specificity: 0.9997 - miss_rate: 1.0000 - fall_out: 3.1190e-04 - mcc: -0.0059 - val_loss: 0.2744 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8393 - val_AUPRC: 0.4249 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 5/1000
198/198 [==============================] - 5s 24ms/step - loss: 0.2858 - accuracy: 0.8865 - recall: 3.4831e-04 - precision: 0.2222 - AUROC: 0.8290 - AUPRC: 0.3511 - f1_score: 6.9553e-04 - balanced_accuracy: 0.5001 - specificity: 0.9998 - miss_rate: 0.9997 - fall_out: 1.5595e-04 - mcc: 0.0046 - val_loss: 0.2725 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8417 - val_AUPRC: 0.4181 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 6/1000
198/198 [==============================] - 5s 23ms/step - loss: 0.2799 - accuracy: 0.8868 - recall: 0.0037 - precision: 0.6364 - AUROC: 0.8373 - AUPRC: 0.3729 - f1_score: 0.0073 - balanced_accuracy: 0.5017 - specificity: 0.9997 - miss_rate: 0.9963 - fall_out: 2.6734e-04 - mcc: 0.0421 - val_loss: 0.2727 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8413 - val_AUPRC: 0.4236 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 7/1000
198/198 [==============================] - 5s 23ms/step - loss: 0.2721 - accuracy: 0.8867 - recall: 0.0019 - precision: 0.6471 - AUROC: 0.8484 - AUPRC: 0.3884 - f1_score: 0.0038 - balanced_accuracy: 0.5009 - specificity: 0.9999 - miss_rate: 0.9981 - fall_out: 1.3367e-04 - mcc: 0.0308 - val_loss: 0.2727 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8414 - val_AUPRC: 0.4215 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
-- Running Boruta Feature Selection:
Iteration: 1 / 100
Confirmed: 0
Tentative: 560
Rejected: 0
Iteration: 2 / 100
Confirmed: 0
Tentative: 560
Rejected: 0
Iteration: 3 / 100
Confirmed: 0
Tentative: 560
Rejected: 0
Iteration: 4 / 100
Confirmed: 0
Tentative: 560
Rejected: 0
Iteration: 5 / 100
Confirmed: 0
Tentative: 560
Rejected: 0
Iteration: 6 / 100
Confirmed: 0
Tentative: 560
Rejected: 0
Iteration: 7 / 100
Confirmed: 0
Tentative: 560
Rejected: 0
Iteration: 8 / 100
Confirmed: 288
Tentative: 181
Rejected: 91
Iteration: 9 / 100
Confirmed: 288
Tentative: 181
Rejected: 91
Iteration: 10 / 100
Confirmed: 288
Tentative: 181
Rejected: 91
Iteration: 11 / 100
Confirmed: 288
Tentative: 181
Rejected: 91
Iteration: 12 / 100
Confirmed: 301
Tentative: 168
Rejected: 91
Iteration: 13 / 100
Confirmed: 301
Tentative: 150
Rejected: 109
Iteration: 14 / 100
Confirmed: 301
Tentative: 150
Rejected: 109
Iteration: 15 / 100
Confirmed: 301
Tentative: 150
Rejected: 109
Iteration: 16 / 100
Confirmed: 306
Tentative: 145
Rejected: 109
Iteration: 17 / 100
Confirmed: 306
Tentative: 133
Rejected: 121
Iteration: 18 / 100
Confirmed: 306
Tentative: 133
Rejected: 121
Iteration: 19 / 100
Confirmed: 312
Tentative: 127
Rejected: 121
Iteration: 20 / 100
Confirmed: 312
Tentative: 127
Rejected: 121
Iteration: 21 / 100
Confirmed: 312
Tentative: 117
Rejected: 131
Iteration: 22 / 100
Confirmed: 314
Tentative: 115
Rejected: 131
Iteration: 23 / 100
Confirmed: 314
Tentative: 115
Rejected: 131
Iteration: 24 / 100
Confirmed: 314
Tentative: 115
Rejected: 131
Iteration: 25 / 100
Confirmed: 314
Tentative: 111
Rejected: 135
Iteration: 26 / 100
Confirmed: 316
Tentative: 109
Rejected: 135
Iteration: 27 / 100
Confirmed: 316
Tentative: 102
Rejected: 142
Iteration: 28 / 100
Confirmed: 316
Tentative: 102
Rejected: 142
Iteration: 29 / 100
Confirmed: 319
Tentative: 99
Rejected: 142
Iteration: 30 / 100
Confirmed: 319
Tentative: 99
Rejected: 142
Iteration: 31 / 100
Confirmed: 319
Tentative: 99
Rejected: 142
Iteration: 32 / 100
Confirmed: 320
Tentative: 98
Rejected: 142
Iteration: 33 / 100
Confirmed: 320
Tentative: 90
Rejected: 150
Iteration: 34 / 100
Confirmed: 320
Tentative: 90
Rejected: 150
Iteration: 35 / 100
Confirmed: 320
Tentative: 90
Rejected: 150
Iteration: 36 / 100
Confirmed: 320
Tentative: 90
Rejected: 150
Iteration: 37 / 100
Confirmed: 321
Tentative: 89
Rejected: 150
Iteration: 38 / 100
Confirmed: 321
Tentative: 89
Rejected: 150
Iteration: 39 / 100
Confirmed: 321
Tentative: 89
Rejected: 150
Iteration: 40 / 100
Confirmed: 322
Tentative: 88
Rejected: 150
Iteration: 41 / 100
Confirmed: 322
Tentative: 88
Rejected: 150
Iteration: 42 / 100
Confirmed: 322
Tentative: 88
Rejected: 150
Iteration: 43 / 100
Confirmed: 322
Tentative: 88
Rejected: 150
Iteration: 44 / 100
Confirmed: 322
Tentative: 86
Rejected: 152
Iteration: 45 / 100
Confirmed: 322
Tentative: 86
Rejected: 152
Iteration: 46 / 100
Confirmed: 325
Tentative: 83
Rejected: 152
Iteration: 47 / 100
Confirmed: 325
Tentative: 83
Rejected: 152
Iteration: 48 / 100
Confirmed: 325
Tentative: 82
Rejected: 153
Iteration: 49 / 100
Confirmed: 325
Tentative: 82
Rejected: 153
Iteration: 50 / 100
Confirmed: 325
Tentative: 78
Rejected: 157
Iteration: 51 / 100
Confirmed: 325
Tentative: 78
Rejected: 157
Iteration: 52 / 100
Confirmed: 325
Tentative: 78
Rejected: 157
Iteration: 53 / 100
Confirmed: 325
Tentative: 78
Rejected: 157
Iteration: 54 / 100
Confirmed: 325
Tentative: 78
Rejected: 157
Iteration: 55 / 100
Confirmed: 325
Tentative: 78
Rejected: 157
Iteration: 56 / 100
Confirmed: 325
Tentative: 75
Rejected: 160
Iteration: 57 / 100
Confirmed: 326
Tentative: 74
Rejected: 160
Iteration: 58 / 100
Confirmed: 326
Tentative: 74
Rejected: 160
Iteration: 59 / 100
Confirmed: 326
Tentative: 74
Rejected: 160
Iteration: 60 / 100
Confirmed: 326
Tentative: 74
Rejected: 160
Iteration: 61 / 100
Confirmed: 326
Tentative: 71
Rejected: 163
Iteration: 62 / 100
Confirmed: 326
Tentative: 71
Rejected: 163
Iteration: 63 / 100
Confirmed: 326
Tentative: 71
Rejected: 163
Iteration: 64 / 100
Confirmed: 326
Tentative: 71
Rejected: 163
Iteration: 65 / 100
Confirmed: 327
Tentative: 70
Rejected: 163
Iteration: 66 / 100
Confirmed: 327
Tentative: 70
Rejected: 163
Iteration: 67 / 100
Confirmed: 329
Tentative: 68
Rejected: 163
Iteration: 68 / 100
Confirmed: 329
Tentative: 66
Rejected: 165
Iteration: 69 / 100
Confirmed: 329
Tentative: 66
Rejected: 165
Iteration: 70 / 100
Confirmed: 329
Tentative: 66
Rejected: 165
Iteration: 71 / 100
Confirmed: 329
Tentative: 66
Rejected: 165
Iteration: 72 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 73 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 74 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 75 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 76 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 77 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 78 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 79 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 80 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 81 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 82 / 100
Confirmed: 331
Tentative: 64
Rejected: 165
Iteration: 83 / 100
Confirmed: 332
Tentative: 63
Rejected: 165
Iteration: 84 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 85 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 86 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 87 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 88 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 89 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 90 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 91 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 92 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 93 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 94 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 95 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 96 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 97 / 100
Confirmed: 332
Tentative: 60
Rejected: 168
Iteration: 98 / 100
Confirmed: 333
Tentative: 59
Rejected: 168
Iteration: 99 / 100
Confirmed: 333
Tentative: 59
Rejected: 168
BorutaPy finished running.
Iteration: 100 / 100
Confirmed: 333
Tentative: 24
Rejected: 168
-- 333 Kept (correlated) features [Boruta]
-- 227 Discarted (uncorrelated) features [Boruta]
Model: "MLP"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense_8 (Dense) (None, 136) 45424
dropout_6 (Dropout) (None, 136) 0
dense_9 (Dense) (None, 8) 1096
dropout_7 (Dropout) (None, 8) 0
dense_10 (Dense) (None, 8) 72
dropout_8 (Dropout) (None, 8) 0
dense_11 (Dense) (None, 1) 9
=================================================================
Total params: 46,601
Trainable params: 46,601
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
198/198 [==============================] - 6s 26ms/step - loss: 0.3990 - accuracy: 0.8602 - recall: 0.1010 - precision: 0.2324 - AUROC: 0.7168 - AUPRC: 0.2418 - f1_score: 0.1408 - balanced_accuracy: 0.5292 - specificity: 0.9573 - miss_rate: 0.8990 - fall_out: 0.0427 - mcc: 0.0854 - val_loss: 0.2760 - val_accuracy: 0.8971 - val_recall: 0.1129 - val_precision: 0.8438 - val_AUROC: 0.8311 - val_AUPRC: 0.4788 - val_f1_score: 0.1991 - val_balanced_accuracy: 0.5551 - val_specificity: 0.9973 - val_miss_rate: 0.8871 - val_fall_out: 0.0027 - val_mcc: 0.2859
Epoch 2/1000
198/198 [==============================] - 4s 20ms/step - loss: 0.3121 - accuracy: 0.8969 - recall: 0.1768 - precision: 0.6731 - AUROC: 0.7986 - AUPRC: 0.4237 - f1_score: 0.2800 - balanced_accuracy: 0.5829 - specificity: 0.9890 - miss_rate: 0.8232 - fall_out: 0.0110 - mcc: 0.3092 - val_loss: 0.2629 - val_accuracy: 0.9091 - val_recall: 0.2613 - val_precision: 0.8065 - val_AUROC: 0.8454 - val_AUPRC: 0.5531 - val_f1_score: 0.3947 - val_balanced_accuracy: 0.6267 - val_specificity: 0.9920 - val_miss_rate: 0.7387 - val_fall_out: 0.0080 - val_mcc: 0.4269
Epoch 3/1000
198/198 [==============================] - 3s 14ms/step - loss: 0.2897 - accuracy: 0.9026 - recall: 0.2271 - precision: 0.7261 - AUROC: 0.8156 - AUPRC: 0.4706 - f1_score: 0.3460 - balanced_accuracy: 0.6081 - specificity: 0.9890 - miss_rate: 0.7729 - fall_out: 0.0110 - mcc: 0.3705 - val_loss: 0.2506 - val_accuracy: 0.9126 - val_recall: 0.3247 - val_precision: 0.7728 - val_AUROC: 0.8492 - val_AUPRC: 0.5787 - val_f1_score: 0.4573 - val_balanced_accuracy: 0.6563 - val_specificity: 0.9878 - val_miss_rate: 0.6753 - val_fall_out: 0.0122 - val_mcc: 0.4652
Epoch 4/1000
198/198 [==============================] - 4s 21ms/step - loss: 0.2814 - accuracy: 0.9035 - recall: 0.2278 - precision: 0.7436 - AUROC: 0.8220 - AUPRC: 0.4748 - f1_score: 0.3488 - balanced_accuracy: 0.6089 - specificity: 0.9900 - miss_rate: 0.7722 - fall_out: 0.0100 - mcc: 0.3770 - val_loss: 0.2545 - val_accuracy: 0.9099 - val_recall: 0.2488 - val_precision: 0.8520 - val_AUROC: 0.8454 - val_AUPRC: 0.5700 - val_f1_score: 0.3851 - val_balanced_accuracy: 0.6216 - val_specificity: 0.9945 - val_miss_rate: 0.7512 - val_fall_out: 0.0055 - val_mcc: 0.4311
Epoch 5/1000
198/198 [==============================] - 5s 23ms/step - loss: 0.2744 - accuracy: 0.9044 - recall: 0.2356 - precision: 0.7504 - AUROC: 0.8294 - AUPRC: 0.4913 - f1_score: 0.3586 - balanced_accuracy: 0.6128 - specificity: 0.9900 - miss_rate: 0.7644 - fall_out: 0.0100 - mcc: 0.3860 - val_loss: 0.2513 - val_accuracy: 0.9121 - val_recall: 0.2892 - val_precision: 0.8185 - val_AUROC: 0.8524 - val_AUPRC: 0.5814 - val_f1_score: 0.4274 - val_balanced_accuracy: 0.6405 - val_specificity: 0.9918 - val_miss_rate: 0.7108 - val_fall_out: 0.0082 - val_mcc: 0.4543
MLP_metrics_estimate = model_metrics_holdout_estimate(MLP_metrics_enhancers["boruta_False"], number_of_splits)
print("-- ENHANCERS -- withOUT BORUTA FEATURE SELECTION")
print(f"MLP Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {MLP_metrics_estimate['accuracy_train']} -- test - {MLP_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {MLP_metrics_estimate['AUROC_train']} -- test - {MLP_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {MLP_metrics_estimate['AUPRC_train']} -- test - {MLP_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("MLP - Train history:")
plot_train_history(MLP_history_enhancers["boruta_False"])
print("-"*100)
MLP_metrics_estimate = model_metrics_holdout_estimate(MLP_metrics_enhancers["boruta_True"], number_of_splits)
print("-- ENHANCERS -- with BORUTA FEATURE SELECTION")
print(f"MLP Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {MLP_metrics_estimate['accuracy_train']} -- test - {MLP_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {MLP_metrics_estimate['AUROC_train']} -- test - {MLP_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {MLP_metrics_estimate['AUPRC_train']} -- test - {MLP_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("MLP - Train history:")
plot_train_history(MLP_history_enhancers["boruta_True"])
-- ENHANCERS -- withOUT BORUTA FEATURE SELECTION MLP Metrics - 10-holdouts estimate: Accuracy : train - 0.9105297463853992 -- test - 0.9092043928261042 AUROC : train - 0.8975376189283605 -- test - 0.8528177944393699 AUPRC : train - 0.6178753437988747 -- test - 0.5708571083326558 -------------------------------------------------------------------------------- MLP - Train history:
---------------------------------------------------------------------------------------------------- -- ENHANCERS -- with BORUTA FEATURE SELECTION MLP Metrics - 10-holdouts estimate: Accuracy : train - 0.9148712175080982 -- test - 0.9137078296594769 AUROC : train - 0.8841010755246934 -- test - 0.8576130727534329 AUPRC : train - 0.62724966027985 -- test - 0.5990892730926193 -------------------------------------------------------------------------------- MLP - Train history:
MLP_metrics_enhancers
{'boruta_True': [{'accuracy_train': 0.9139606541834558,
'accuracy_test': 0.9145137078296595,
'AUROC_train': 0.8870782490688551,
'AUROC_test': 0.858294713532465,
'AUPRC_train': 0.6307042609597342,
'AUPRC_test': 0.6038708123262326},
{'accuracy_train': 0.9159160938611045,
'accuracy_test': 0.9155408074583234,
'AUROC_train': 0.8833191924748005,
'AUROC_test': 0.8643202097423118,
'AUPRC_train': 0.6211444423614926,
'AUPRC_test': 0.6046434026438599},
{'accuracy_train': 0.9147902346527613,
'accuracy_test': 0.9160938611045272,
'AUROC_train': 0.8826625985722133,
'AUROC_test': 0.8694618646672756,
'AUPRC_train': 0.6258925511772357,
'AUPRC_test': 0.6290504574729222},
{'accuracy_train': 0.9197479655526586,
'accuracy_test': 0.9163308840957572,
'AUROC_train': 0.9010941014966156,
'AUROC_test': 0.8658580674968345,
'AUPRC_train': 0.6772079423458085,
'AUPRC_test': 0.613083650673553},
{'accuracy_train': 0.9166074109188591,
'accuracy_test': 0.9133285928735088,
'AUROC_train': 0.8834717597906181,
'AUROC_test': 0.840190622327844,
'AUPRC_train': 0.6422991227386761,
'AUPRC_test': 0.5832580470684916},
{'accuracy_train': 0.9082128466461247,
'accuracy_test': 0.9062969108003477,
'AUROC_train': 0.8815763780260043,
'AUROC_test': 0.8523450390192984,
'AUPRC_train': 0.6059693802934278,
'AUPRC_test': 0.5744280393407555},
{'accuracy_train': 0.9164691475073082,
'accuracy_test': 0.9145927154934028,
'AUROC_train': 0.8727694043067702,
'AUROC_test': 0.8553737773673789,
'AUPRC_train': 0.6132365320830765,
'AUPRC_test': 0.595887105237059},
{'accuracy_train': 0.9101682863237734,
'accuracy_test': 0.9114324089436675,
'AUROC_train': 0.879986656625982,
'AUROC_test': 0.8600656872979098,
'AUPRC_train': 0.6058947108736727,
'AUPRC_test': 0.5979999491116564},
{'accuracy_train': 0.9168246819941535,
'accuracy_test': 0.9168049300782176,
'AUROC_train': 0.8874548329431736,
'AUROC_test': 0.8580280024864052,
'AUPRC_train': 0.6329515771918617,
'AUPRC_test': 0.6110071370294059},
{'accuracy_train': 0.9160148534407837,
'accuracy_test': 0.912143477917358,
'AUROC_train': 0.8815975819419025,
'AUROC_test': 0.8521927435966061,
'AUPRC_train': 0.6171960827735131,
'AUPRC_test': 0.5776641300222563}],
'boruta_False': [{'accuracy_train': 0.9141384214268784,
'accuracy_test': 0.9137236311922257,
'AUROC_train': 0.8886063277948006,
'AUROC_test': 0.8565762747018207,
'AUPRC_train': 0.6329700412279043,
'AUPRC_test': 0.601920638299909},
{'accuracy_train': 0.9152840325511574,
'accuracy_test': 0.9153827921308367,
'AUROC_train': 0.8975677350848473,
'AUROC_test': 0.8545910006290531,
'AUPRC_train': 0.6418890095476121,
'AUPRC_test': 0.6052779145006851},
{'accuracy_train': 0.9136643754444181,
'accuracy_test': 0.9141976771746859,
'AUROC_train': 0.8964363888032585,
'AUROC_test': 0.8668308642120971,
'AUPRC_train': 0.6354552875045827,
'AUPRC_test': 0.6208483281944311},
{'accuracy_train': 0.918286323773406,
'accuracy_test': 0.914829738484633,
'AUROC_train': 0.9014334281701266,
'AUROC_test': 0.8603929439248563,
'AUPRC_train': 0.671757679081043,
'AUPRC_test': 0.6014387488514324},
{'accuracy_train': 0.9206170498538359,
'accuracy_test': 0.9149087461483764,
'AUROC_train': 0.8987534704777007,
'AUROC_test': 0.8417249094455453,
'AUPRC_train': 0.6657446368778149,
'AUPRC_test': 0.5892876075514842},
{'accuracy_train': 0.9148099865686972,
'accuracy_test': 0.9123014932448448,
'AUROC_train': 0.9049254473420982,
'AUROC_test': 0.852215813015375,
'AUPRC_train': 0.6406039519730566,
'AUPRC_test': 0.5869230337303645},
{'accuracy_train': 0.9188196255036739,
'accuracy_test': 0.9163308840957572,
'AUROC_train': 0.9065788968882552,
'AUROC_test': 0.8541509118785462,
'AUPRC_train': 0.6684138853538305,
'AUPRC_test': 0.6062787354197494},
{'accuracy_train': 0.8865844986963736,
'accuracy_test': 0.8866240025282452,
'AUROC_train': 0.8934875410135724,
'AUROC_test': 0.8437537142385197,
'AUPRC_train': 0.5053385067353211,
'AUPRC_test': 0.4661779104964656},
{'accuracy_train': 0.9163901398435649,
'accuracy_test': 0.9171209607331912,
'AUROC_train': 0.8961874668584541,
'AUROC_test': 0.8563968424392853,
'AUPRC_train': 0.6421917632428289,
'AUPRC_test': 0.610069493949375},
{'accuracy_train': 0.8867030101919886,
'accuracy_test': 0.8866240025282452,
'AUROC_train': 0.8913994868504914,
'AUROC_test': 0.841544669908598,
'AUPRC_train': 0.4743886764447526,
'AUPRC_test': 0.42034867233266104}]}
# Saving results
import json
# Saving metrics
file = open("results/MLP_metrics_enhancers.txt", "w")
string = repr(MLP_metrics_enhancers)
file.write("MLP_metrics_enhancers = " + string + "\n")
file.close()
# Saving history
for boruta in ("boruta_True", "boruta_False"):
with open("results/MLP_history_enhancers_" + boruta + ".json", 'w') as h:
for holdout in range(number_of_splits):
json.dump(MLP_history_enhancers[boruta][holdout].history, h)
MLP_distinct_metrics_enhancers = {}
MLP_distinct_metrics_enhancers["boruta_True"] = {}
MLP_distinct_metrics_enhancers["boruta_False"] = {}
for boruta in ("boruta_True", "boruta_False"):
MLP_distinct_metrics_enhancers[boruta]["accuracy_test"] = []
MLP_distinct_metrics_enhancers[boruta]["AUROC_test"] = []
MLP_distinct_metrics_enhancers[boruta]["AUPRC_test"] = []
for holdout in MLP_metrics_enhancers[boruta]:
MLP_distinct_metrics_enhancers[boruta]["accuracy_test"].append(holdout['accuracy_test'])
MLP_distinct_metrics_enhancers[boruta]["AUROC_test"].append(holdout['AUROC_test'])
MLP_distinct_metrics_enhancers[boruta]["AUPRC_test"].append(holdout['AUPRC_test'])
print("MLP - Enhancers - Wilcoxon test - Boruta model performances VS Non-Boruta model performances")
print(f"-Accuracy: {wilcoxon(MLP_distinct_metrics_enhancers['boruta_True']['accuracy_test'], MLP_distinct_metrics_enhancers['boruta_False']['accuracy_test'])}")
print(f"-AUROC: {wilcoxon(MLP_distinct_metrics_enhancers['boruta_True']['AUROC_test'], MLP_distinct_metrics_enhancers['boruta_False']['AUROC_test'])}")
print(f"-AUPRC: {wilcoxon(MLP_distinct_metrics_enhancers['boruta_True']['AUPRC_test'], MLP_distinct_metrics_enhancers['boruta_False']['AUPRC_test'])}")
MLP - Enhancers - Wilcoxon test - Boruta model performances VS Non-Boruta model performances -Accuracy: WilcoxonResult(statistic=21.0, pvalue=0.556640625) -AUROC: WilcoxonResult(statistic=3.0, pvalue=0.009765625) -AUPRC: WilcoxonResult(statistic=19.0, pvalue=0.431640625)
Load and visualize genomic sequence data for both promoters and enhancers for cell line HepG2.
genome = Genome("hg38")
genome
C:\Users\Paolo\anaconda3\lib\site-packages\dateparser\date_parser.py:35: PytzUsageWarning: The localize method is no longer necessary, as this time zone supports the fold attribute (PEP 495). For more details on migrating to a PEP 495-compliant implementation, see https://pytz-deprecation-shim.readthedocs.io/en/latest/migration.html date_obj = stz.localize(date_obj)
Human, Homo sapiens, hg38, 2013-12-04, 25 chromosomes
cell_line = "HEPG2"
window_size = 256
promoters_epigenomes, promoters_labels = load_epigenomes(
cell_line = cell_line,
dataset = "fantom",
region = "promoters",
window_size = window_size,
binarize = True,
min_active_tpm_value = 1,
max_inactive_tpm_value = 1
)
enhancers_epigenomes, enhancers_labels = load_epigenomes(
cell_line = cell_line,
dataset = "fantom",
region = "enhancers",
window_size = window_size,
binarize = True,
min_active_tpm_value = 0.09,
max_inactive_tpm_value = 0.09
)
def to_bed(data:pd.DataFrame)->pd.DataFrame:
"""Return bed coordinates from given dataset."""
return data.reset_index()[data.index.names]
bed_promoters = to_bed(promoters_epigenomes)
bed_enhancers = to_bed(enhancers_epigenomes)
bed_promoters
| chrom | chrom | chromStart | chromEnd | strand |
|---|---|---|---|---|
| 0 | chr1 | 628964 | 629220 | + |
| 1 | chr1 | 629013 | 629269 | + |
| 2 | chr1 | 629642 | 629898 | + |
| 3 | chr1 | 629847 | 630103 | + |
| 4 | chr1 | 629905 | 630161 | + |
| ... | ... | ... | ... | ... |
| 99876 | chrX | 154441877 | 154442133 | + |
| 99877 | chrX | 154584927 | 154585183 | + |
| 99878 | chrY | 1452882 | 1453138 | - |
| 99879 | chrY | 2500959 | 2501215 | - |
| 99880 | chrY | 14055973 | 14056229 | + |
99881 rows × 4 columns
bed_enhancers
| chrom | chrom | chromStart | chromEnd | strand |
|---|---|---|---|---|
| 0 | chr10 | 100006381 | 100006637 | . |
| 1 | chr10 | 100008146 | 100008402 | . |
| 2 | chr10 | 100014418 | 100014674 | . |
| 3 | chr10 | 100020216 | 100020472 | . |
| 4 | chr10 | 100043528 | 100043784 | . |
| ... | ... | ... | ... | ... |
| 63280 | chrY | 7520247 | 7520503 | . |
| 63281 | chrY | 7724272 | 7724528 | . |
| 63282 | chrY | 7770029 | 7770285 | . |
| 63283 | chrY | 7796295 | 7796551 | . |
| 63284 | chrY | 8007562 | 8007818 | . |
63285 rows × 4 columns
promoters_genome = genome.bed_to_sequence(to_bed(promoters_epigenomes))
enhancers_genome = genome.bed_to_sequence(to_bed(enhancers_epigenomes))
print("-- PROMOTERS:")
print(promoters_genome)
print("-"*80)
print("-- ENHANCERS:")
print(enhancers_genome)
-- PROMOTERS: ['AAATGGTCATCCATCCTTTGGCCCCAATACCTAAACTAAGGTCTATGAACAATAAGATGATTTTCTTCAGTGGGACTTTTTTGTTTAATATATTAGATTTGACCTTCAGCAAGGTCAAAGGGAGTCCGAACTAGTCTCAGGCTTCAACATCGAATACGCCGCAGGCCCCTTCGCCCTATTCTTCATAGCCGAATACACAAACATTATTATAATAAACACCCTCACCACTACAATCTTCCTAGGAACAACATATAAC' 'CAATAAGATGATTTTCTTCAGTGGGACTTTTTTGTTTAATATATTAGATTTGACCTTCAGCAAGGTCAAAGGGAGTCCGAACTAGTCTCAGGCTTCAACATCGAATACGCCGCAGGCCCCTTCGCCCTATTCTTCATAGCCGAATACACAAACATTATTATAATAAACACCCTCACCACTACAATCTTCCTAGGAACAACATATAACGCACTCTCCCCTGAACTCTACACAACATATTTTGTCACCAAGACCCTAC' 'AATCCCCTGGCCCAACCCGTCATCTACTCTACCATCTTTGCAGGCACACTCATCACAGCGCTAAGCTCGCACTGATTTTTTACCTGAGTAGGCCTAGAAATAAACATGCTAGCTTTTATTCCAGTTCTAACCAAAAAAATAAACCCTCGTTCCACAGAAGCTGCCATCAAGTATTTCCTCACGCAAGCAACCGCATCCATAATCCTTCTAATAGCTATCCTCTTCAACAATATACTCTCCGGACAATGAACCATAA' ... 'AGATTTATAACATATATGATATAATCATGGGGGGGGTGAATAATTGAGACTTTATTCTTATAGAATTGTATTTTTATTTATTCTTATAGAATTGTATTTGACTTATATTCCTGACAATTCTTTTATAGAGTTATATTTGAATTGTATTCTTGCCAGGTGTCTTACTGAATTCTATTTGAATTGTGTTCTTGCCATTTCTCTTGTTGAATTGTATTTGCACTGTATTTTGCCAGCTCTTTTCAGAGTTGTATTTGAG' 'TATTTATAAAGTCTCATATTTATTTATTTAAGTAAGTAATAAGGGTATATATATTTTAAGTTATTTTTCATTTATTTTAGGTTATTTTAATTCATAAAAGGTTTTTGGTTTTGAGTTTTATTTAAATTTCTCAAAGCTTTATGTTATTCATGAAGTCAAAAGTTTTAGATAAGTGGTTAGTTTTGATTGAAGTGTTTTTTTTGAAGTTGTGAGTCTTGATTGAAGGTTTTTTTTCATGGAAGTTGCGAGTCCTGAT' 'CATGTCAGTTTGGGAACTATCTTTAGGGGTGCTAACCCAAATCTCCCTTTCTCCAACTAGAGCACCCTGCCTTCCTCATTAGATGTCACCGCCCCCCCCCCCCGCCCCCGACTTCATCCGCCATGTCCTGATGGTGCTTTATGACGTATAAGGCCTTCCTTCCCACCCAGGGCTACCATTGGCTGGGTAGTGGAGTGTTGACCAATCACAGCTCAGGGGCGTGATCGTCTCGTCCTGGGATCGCGAGAGGGGTATA'] -------------------------------------------------------------------------------- -- ENHANCERS: ['TTCTTGAGCTAGAGAGGGCCCAGTTattttacagatgaggaaacgaagactcagggaagaattatggtgtgagtcacacagctctagcttgaaaaggtctccctgactcagcccttgtcctcttcctgtctattctcaacccagcagccagttatccttttaaaacataagtcatcatcaacttcactgttcaaagctctccaacggcttccccctcactcagtgtaaaggccaaaatccttacaatgacctacac' 'caggcggatcacctgaagtcaggagCCAGAAAAGAGGAAGAGATTGATGGAACGTTGAGGGAAATGCCCTGCCTTGGGGCTGCAAGACAAATAAATAACTGGTGCATATTTAAACAGATCGTTTATGAAGATTATTATTATCTCAGCCAAAAAATACACAATATGAACACTGTATAAAAGTCAATTTATCACATTCTCTGAGTCAGACTGTTCTTGGCTTATTGCAAAAGTTATCAGTGCACTGCACGAACCCCGT' 'gaaagaaTCAGCCATGTAaagagctcttcaggcaggagaaactgcaggtgcaaaatcttgaagtagaaagccatctagtacgtttcggaaacacaaaataagccatgtgaacaaaggaggcagttttatgagatcagattagaaaacaaacagggccaggtgcagtgtcacgcatctgtaatcccagctactttggagggtgaggtgagaggctcacttgagcccagaagttcaagattgcagcacattatgattg' ... 'GATTCAGGTCCTCAGGAGGAGAAACAAAGGGTACATTCTCAGCATTCTCTGAGATTTACTTTCCCAGAAATGAAACAGCAAAGGCCAAAAGAATTGTTTTTCTTGCATCACTTGACCCTGAATTATCAGGGCTGATTCATTCAGTCACCTGTGCTGCTCATAAGTACTTGGGAGTACAGCTTGCTGATATGGCCTAAGAGGAAACTTTTCTCAGACTCCAAGCATATGGTCAATTTGGAATGAAGCAGCTGACTGT' 'ACAAAAATCAGGGGAGGCATTTGAAAACATGGTGCTCAGCTTGATGAATTCCCTAAAAGTTACAACCCAGCACTAGTAGATTTTCCtccctttccctttccctttccctttccctttccctttccctttccctttccctttccctttccctttccctcccctcccctcccctcccctctcctttcctttccttttcttttctttctaacagggtctcactctgtcaccaggctggagtgcagtgatgccatcacag' 'tcacaatgtctgtgatgtgcagtaccaatgctggagagtgtcttcacattgtttccagccccagcaacatgtcacaatcttcctgtggtcaggtcccaatcaaaagtgaagaaacatcacataggtgctgagccaagtgagatgttacatcactttctgttgacagaacccaaaaaggagaatcacatcacctatgtgcagtacccagttattgtcacagtgcactgtgagtgaaggactaatgcagtagaaggga']
Check for NaN values, n/N represent a missing value. In this case NaN values are ignored as there are very few.
promoters_NaN = sum([
nucleotide.lower() == "n"
for sequence in (promoters_genome)
for nucleotide in sequence
])
enhancers_NaN = sum([
nucleotide.lower() == "n"
for sequence in (enhancers_genome)
for nucleotide in sequence
])
print(f"Promoters NaN values: {promoters_NaN}")
print(f"Enhancers NaN values: {enhancers_NaN}")
Promoters NaN values: 4 Enhancers NaN values: 1
The following step constructs the Mixed Sequence to train CNNs with DNA sequence data.
def get_cnn_sequence(
genome: Genome,
bed: pd.DataFrame,
y: np.ndarray,
batch_size: int
) -> MixedSequence:
""" Returns the genomic sequence data for the CNN training.
Parameters
-------------------------
genome: Genome,
The genome from where to extract the genomic sequence.
bed: pd.DataFrame,
The BED file coordinates describing where to extract the sequences.
y: np.ndarray,
The values the model should predict.
batch_size: int,
The size of the batches to generate.
Returns
--------------------------
MixedSequence object to train a CNN."""
return MixedSequence(
x = BedSequence(
genome,
bed,
batch_size
),
y = VectorSequence(y, batch_size)
)
batch_size=256
mixed_sequence_promoters = get_cnn_sequence(genome, bed_promoters, promoters_labels[cell_line].values, batch_size)
mixed_sequence_enhancers = get_cnn_sequence(genome, bed_enhancers, enhancers_labels[cell_line].values, batch_size)
mixed_sequence_promoters[0]
(array([[[0., 0., 0., 1.],
[0., 0., 1., 0.],
[0., 0., 0., 1.],
...,
[0., 1., 0., 0.],
[0., 0., 1., 0.],
[0., 1., 0., 0.]],
[[1., 0., 0., 0.],
[0., 1., 0., 0.],
[0., 0., 1., 0.],
...,
[0., 0., 1., 0.],
[0., 0., 0., 1.],
[1., 0., 0., 0.]],
[[0., 0., 1., 0.],
[0., 0., 0., 1.],
[0., 0., 0., 1.],
...,
[0., 1., 0., 0.],
[0., 1., 0., 0.],
[0., 0., 1., 0.]],
...,
[[0., 1., 0., 0.],
[0., 0., 1., 0.],
[0., 1., 0., 0.],
...,
[0., 1., 0., 0.],
[0., 0., 0., 1.],
[0., 0., 0., 1.]],
[[0., 0., 1., 0.],
[0., 0., 1., 0.],
[0., 0., 1., 0.],
...,
[0., 1., 0., 0.],
[1., 0., 0., 0.],
[0., 1., 0., 0.]],
[[0., 0., 0., 1.],
[0., 0., 0., 1.],
[0., 0., 0., 1.],
...,
[0., 1., 0., 0.],
[1., 0., 0., 0.],
[0., 1., 0., 0.]]]),
array([False, False, False, False, False, False, False, True, False,
False, True, True, False, False, False, False, False, True,
False, False, False, False, False, False, False, False, False,
False, True, True, False, False, False, False, False, True,
True, False, True, True, False, True, False, False, False,
False, False, False, False, False, False, False, False, False,
True, True, False, False, False, True, True, False, False,
False, False, False, True, False, False, False, False, True,
False, False, False, False, False, False, False, False, False,
True, True, False, False, True, False, False, False, False,
False, False, False, False, True, True, False, True, False,
False, False, False, False, False, False, False, False, False,
True, False, False, False, False, False, False, True, False,
False, False, True, True, False, False, True, False, False,
False, True, False, False, False, False, False, False, False,
False, True, False, True, False, False, False, False, False,
False, True, False, True, False, False, False, False, False,
False, False, False, False, False, False, True, False, False,
False, False, True, True, False, True, False, True, True,
False, False, True, False, False, False, False, False, False,
True, True, False, False, False, True, False, True, True,
True, False, True, True, False, False, False, False, False,
True, True, True, False, True, True, False, False, True,
True, False, True, False, True, True, False, False, False,
False, False, False, False, False, False, False, False, False,
False, False, True, False, False, False, False, True, False,
False, False, False, True, False, False, False, True, True,
False, False, False, False, True, False, True, False, True,
False, False, False, False]))
A convolutional neural network consists a set of convolutional layers (convolution+maxPooling) and a set of dense layer. To avoid overfitting of the model dropout layers were added.
To have a statistically sound estimate of an architecture performance, multiple models are built and trained, each with the same architecture, over different portions of the data (holdouts) and, the average performance of those, is considered as an estimate of the overall performance of the architecture.
Hyperparameter tuning is performed training a meta model over training and validation data to obtain the best set of hyperparameters via Bayesian optimization.
Here are the functions to evaluate the model performances, build and train the hypermodel for hyperparameter tuning, and to build and train the model with the best set of hyperparameters found.
def model_metrics_holdout_estimate_evaluation(
model_metrics: list,
holdout_total_number: int
) -> dict[str, float]:
""" Return the average metrics (accuracy, AUROC, AUPRC) of the model accross all the holdouts for train and test.
Parameters
------------------------
model_metrics: list
The list with the train/test metrics (accuracy, AUROC, AUPRC) for all the holdouts.
holdout_total_number: int
The number of total holdouts.
Returns
------------------------
dict: dict[str, float]
The average metrics of the model accross all the holdouts for train and test."""
total_accuracy_train = 0
total_accuracy_test = 0
total_AUROC_train = 0
total_AUROC_test = 0
total_AUPRC_train = 0
total_AUPRC_test = 0
for holdout in model_metrics:
total_accuracy_train += holdout["train_evaluation"]['accuracy']
total_AUROC_train += holdout["train_evaluation"]['AUROC']
total_AUPRC_train += holdout["train_evaluation"]['AUPRC']
for holdout in model_metrics:
total_accuracy_test += holdout["test_evaluation"]['accuracy']
total_AUROC_test += holdout["test_evaluation"]['AUROC']
total_AUPRC_test += holdout["test_evaluation"]['AUPRC']
avg_accuracy_train = total_accuracy_train / holdout_total_number
avg_accuracy_test = total_accuracy_test / holdout_total_number
avg_AUROC_train = total_AUROC_train / holdout_total_number
avg_AUROC_test = total_AUROC_test / holdout_total_number
avg_AUPRC_train = total_AUPRC_train / holdout_total_number
avg_AUPRC_test = total_AUPRC_test / holdout_total_number
return {
'accuracy_train': avg_accuracy_train,
'accuracy_test': avg_accuracy_test,
'AUROC_train': avg_AUROC_train,
'AUROC_test': avg_AUROC_test,
'AUPRC_train': avg_AUPRC_train,
'AUPRC_test': avg_AUPRC_test,
}
def hyperparameter_tuning_CNN(
x_train: pd.DataFrame,
x_validation: pd.DataFrame,
y_train: np.ndarray,
y_validation: np.ndarray,
hypermodel: tf.keras.Model,
name: str,
directory: str,
max_trials: int = 10,
epochs: int = 50,
batch_size: int = 256
):
""" Tunes the hypermodel with bayesian optiomization and returns the best set of hyperparameters.
Parameters
------------------------
x_train: pd.DataFrame
The input data for training the hypermodel.
x_validation: pd.DataFrame
The input data for testing the hypermodel.
y_train: np.ndarray
The labels of the input data for training the hypermodel.
y_validation: np.ndarray
The labels of the input data for testing the hypermodel.
hypermodel: model
The hypermodel to tune or a function that builds a model using the hp object.
name: str
The project name.
directory: str
The directory where to store results.
max_trials: int
The maximum amount of model configuration that the algorithm will try out.
epochs: int
The number of times the learning algorithm works through the dataset.
batch_size: int
The number of samples to work through before updating the model.
Returns
------------------------
hyperparameters: hyperparameters
The best set of hyperparameters."""
train_sequence = get_cnn_sequence(genome, x_train, y_train, batch_size)
validation_sequence = get_cnn_sequence(genome, x_validation, y_validation, batch_size)
tuner = kt.tuners.BayesianOptimization(
hypermodel,
objective = [
kt.Objective("val_accuracy", direction="max"),
kt.Objective("val_AUROC", direction="max"),
kt.Objective("val_AUPRC", direction="max")],
max_trials = max_trials,
project_name = name,
directory = directory
)
tuner.search(
train_sequence,
epochs = epochs,
batch_size = batch_size,
validation_data = validation_sequence,
callbacks = [EarlyStopping("val_loss", patience=2)]
)
best_hp = tuner.get_best_hyperparameters(num_trials=1)[0]
print(f"-- Best set of hyperparameters found:\n{best_hp.values}")
return best_hp
def build_CNN_hypermodel(hp) -> tf.keras.Model:
""" Returns the CNN hypermodel.
Parameters
------------------------
hp: hyperparameters
The hyperparameters object.
Returns
------------------------
hypermodel: model
The CNN hypermodel."""
CNN_Hypermodel = Sequential(name="CNN_Hypermodel")
CNN_Hypermodel.add(layers.Input((window_size, 4)))
for i in range(hp.Int('conv_depth', min_value=1, max_value=6)):
CNN_Hypermodel.add(layers.Conv1D(
filters = hp.Int('filters_' + str(i), min_value=8, max_value=160, step=8),
kernel_size = hp.Choice('kernel_size_' + str(i), [6, 8, 12, 16]),
padding='same',
activation='relu'))
CNN_Hypermodel.add(layers.Dropout(hp.Choice('conv_dropout_'+ str(i), [0.0, 0.1, 0.2])))
CNN_Hypermodel.add(layers.MaxPooling1D(2))
CNN_Hypermodel.add(layers.Flatten())
for i in range(hp.Int('dense_depth', min_value=1, max_value=4)):
CNN_Hypermodel.add(layers.Dense(units = hp.Int('units_' + str(i), min_value=16, max_value=128, step=16), activation="relu"))
CNN_Hypermodel.add(layers.Dropout(hp.Choice('dense_dropout_'+ str(i), [0.3, 0.4, 0.5])))
CNN_Hypermodel.add(layers.Dense(1, activation='sigmoid'))
CNN_Hypermodel.compile(
optimizer='nadam',
loss='binary_crossentropy',
metrics=get_standard_binary_metrics()
)
CNN_Hypermodel.summary()
return CNN_Hypermodel
def build_CNN(
best_hyperparameters: dict
) -> tf.keras.Model:
""" Returns the CNN model with the best set of hyperparameters.
Parameters
------------------------
best_hyperparameters: dict
The best set of hyperparameters for the model.
Returns
------------------------
model: model
The best CNN model."""
CNN = Sequential(name="CNN")
CNN.add(layers.Input((window_size, 4)))
for i in range(best_hyperparameters['conv_depth']):
CNN.add(layers.Conv1D(
filters = best_hyperparameters['filters_' + str(i)],
kernel_size = best_hyperparameters['kernel_size_' + str(i)],
padding='same',
activation='relu'))
CNN.add(layers.Dropout(best_hyperparameters['conv_dropout_' + str(i)]))
CNN.add(layers.MaxPooling1D(2))
CNN.add(layers.Flatten())
for i in range(best_hyperparameters['dense_depth']):
CNN.add(layers.Dense(units = best_hyperparameters['units_' + str(i)], activation='relu'))
CNN.add(layers.Dropout(best_hyperparameters['dense_dropout_' + str(i)]))
CNN.add(layers.Dense(1, activation='sigmoid'))
CNN.compile(
optimizer='nadam',
loss='binary_crossentropy',
metrics=get_standard_binary_metrics()
)
CNN.summary()
return CNN
def train_CNN(
model: tf.keras.Model,
x_train: pd.DataFrame,
x_test: pd.DataFrame,
y_train: np.ndarray,
y_test: np.ndarray,
genome: Genome,
epochs: int,
batch_size: int
):
""" Train a model and return the performance metrics (accuracy, AUROC, AUPRC) for train and test and the training history.
Parameters
------------------------
x_train: pd.DataFrame
The input data for training.
x_test: pd.DataFrame
The input data for testing.
y_train: np.ndarray
The labels of the input data for training.
y_test: np.ndarray
The labels of the input data for testing.
genome: Genome,
The genome from where to extract the genomic sequence.
epochs: int
The number of times the learning algorithm works through the dataset.
batch_size: int
The number of samples to work through before updating the model.
Returns
------------------------
The performance metrics and history of the model."""
train_sequence = get_cnn_sequence(genome, x_train, y_train, batch_size)
test_sequence = get_cnn_sequence(genome, x_test, y_test, batch_size)
CNN_history = model.fit(
train_sequence,
validation_data = test_sequence,
epochs = epochs,
batch_size = batch_size,
callbacks = [EarlyStopping("val_loss", patience=2)]
)
train_evaluation = model.evaluate(train_sequence, return_dict=True)
test_evaluation = model.evaluate(test_sequence, return_dict=True)
CNN_metrics = {"train_evaluation": train_evaluation, "test_evaluation": test_evaluation}
return CNN_metrics, CNN_history
epochs = 1000
batch_size = 256
print("---- PROMOTERS ----")
CNN_metrics_promoters = []
CNN_history_promoters = []
for holdout_number, (train_indices, test_indices) in enumerate(holdouts_generator.split(bed_promoters, promoters_labels)):
print(f"-- HOLDOUT {holdout_number+1} --")
# Train/Test data
x_train_promoters, x_test_promoters = bed_promoters.iloc[train_indices], bed_promoters.iloc[test_indices]
y_train_promoters, y_test_promoters = promoters_labels.iloc[train_indices], promoters_labels.iloc[test_indices]
## Hyperparameter tuning
# Generate holdouts
for holdout_number_tuning, (train_indices_tuning, val_indices_tuning) in enumerate(holdouts_generator_tuning.split(x_train_promoters, y_train_promoters)):
# Train/Validation data
x_train_promoters_tuning, x_val_promoters_tuning = x_train_promoters.iloc[train_indices_tuning], x_train_promoters.iloc[val_indices_tuning]
y_train_promoters_tuning, y_val_promoters_tuning = y_train_promoters.iloc[train_indices_tuning], y_train_promoters.iloc[val_indices_tuning]
hp = kt.HyperParameters()
best_hyperparameters = hyperparameter_tuning_CNN(
x_train_promoters_tuning,
x_val_promoters_tuning,
y_train_promoters_tuning.values,
y_val_promoters_tuning.values,
build_CNN_hypermodel,
name = "CNN_hypermodel_promoters_" + str(holdout_number),
directory='CNN_hypermodel_promoters',
max_trials = 15,
epochs = 50,
batch_size = 256
)
#Build CNN with best set of hyperparameters
CNN = build_CNN(best_hyperparameters)
print("- Training model:\n")
CNN_holdout_metrics, CNN_holdout_history = train_CNN(
CNN,
x_train_promoters,
x_test_promoters,
y_train_promoters.values,
y_test_promoters.values,
genome,
epochs,
batch_size
)
CNN_metrics_promoters.append(CNN_holdout_metrics)
CNN_history_promoters.append(CNN_holdout_history)
Trial 15 Complete [00h 00m 42s]
multi_objective: -2.0165334939956665
Best multi_objective So Far: -2.106355130672455
Total elapsed time: 00h 23m 05s
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 6, 'conv_dropout_0': 0.2, 'dense_depth': 1, 'units_0': 128, 'dense_dropout_0': 0.3, 'filters_1': 8, 'kernel_size_1': 16, 'conv_dropout_1': 0.2, 'units_1': 16, 'dense_dropout_1': 0.5, 'filters_2': 8, 'kernel_size_2': 12, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.1, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.0, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
Model: "CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 160) 4000
dropout (Dropout) (None, 256, 160) 0
max_pooling1d (MaxPooling1D (None, 128, 160) 0
)
conv1d_1 (Conv1D) (None, 128, 8) 20488
dropout_1 (Dropout) (None, 128, 8) 0
max_pooling1d_1 (MaxPooling (None, 64, 8) 0
1D)
conv1d_2 (Conv1D) (None, 64, 8) 776
dropout_2 (Dropout) (None, 64, 8) 0
max_pooling1d_2 (MaxPooling (None, 32, 8) 0
1D)
conv1d_3 (Conv1D) (None, 32, 8) 392
dropout_3 (Dropout) (None, 32, 8) 0
max_pooling1d_3 (MaxPooling (None, 16, 8) 0
1D)
conv1d_4 (Conv1D) (None, 16, 8) 392
dropout_4 (Dropout) (None, 16, 8) 0
max_pooling1d_4 (MaxPooling (None, 8, 8) 0
1D)
conv1d_5 (Conv1D) (None, 8, 8) 392
dropout_5 (Dropout) (None, 8, 8) 0
max_pooling1d_5 (MaxPooling (None, 4, 8) 0
1D)
flatten (Flatten) (None, 32) 0
dense (Dense) (None, 128) 4224
dropout_6 (Dropout) (None, 128) 0
dense_1 (Dense) (None, 1) 129
=================================================================
Total params: 30,793
Trainable params: 30,793
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 12s 33ms/step - loss: 0.5477 - accuracy: 0.7400 - recall: 0.0031 - precision: 0.2319 - AUROC: 0.6425 - AUPRC: 0.3474 - f1_score: 0.0061 - balanced_accuracy: 0.4998 - specificity: 0.9964 - miss_rate: 0.9969 - fall_out: 0.0036 - mcc: -0.0035 - val_loss: 0.5260 - val_accuracy: 0.7419 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.7217 - val_AUPRC: 0.4230 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 2/1000
313/313 [==============================] - 11s 35ms/step - loss: 0.5166 - accuracy: 0.7415 - recall: 0.0169 - precision: 0.4774 - AUROC: 0.7136 - AUPRC: 0.4214 - f1_score: 0.0326 - balanced_accuracy: 0.5052 - specificity: 0.9936 - miss_rate: 0.9831 - fall_out: 0.0064 - mcc: 0.0481 - val_loss: 0.5301 - val_accuracy: 0.7415 - val_recall: 0.0824 - val_precision: 0.4953 - val_AUROC: 0.7348 - val_AUPRC: 0.4393 - val_f1_score: 0.1413 - val_balanced_accuracy: 0.5266 - val_specificity: 0.9708 - val_miss_rate: 0.9176 - val_fall_out: 0.0292 - val_mcc: 0.1148
Epoch 3/1000
313/313 [==============================] - 11s 35ms/step - loss: 0.5090 - accuracy: 0.7438 - recall: 0.0968 - precision: 0.5202 - AUROC: 0.7274 - AUPRC: 0.4433 - f1_score: 0.1632 - balanced_accuracy: 0.5329 - specificity: 0.9689 - miss_rate: 0.9032 - fall_out: 0.0311 - mcc: 0.1345 - val_loss: 0.5081 - val_accuracy: 0.7468 - val_recall: 0.1575 - val_precision: 0.5321 - val_AUROC: 0.7456 - val_AUPRC: 0.4632 - val_f1_score: 0.2430 - val_balanced_accuracy: 0.5546 - val_specificity: 0.9518 - val_miss_rate: 0.8425 - val_fall_out: 0.0482 - val_mcc: 0.1800
Epoch 4/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.5035 - accuracy: 0.7508 - recall: 0.1965 - precision: 0.5482 - AUROC: 0.7371 - AUPRC: 0.4665 - f1_score: 0.2893 - balanced_accuracy: 0.5701 - specificity: 0.9437 - miss_rate: 0.8035 - fall_out: 0.0563 - mcc: 0.2117 - val_loss: 0.5057 - val_accuracy: 0.7529 - val_recall: 0.3174 - val_precision: 0.5362 - val_AUROC: 0.7530 - val_AUPRC: 0.4850 - val_f1_score: 0.3988 - val_balanced_accuracy: 0.6109 - val_specificity: 0.9045 - val_miss_rate: 0.6826 - val_fall_out: 0.0955 - val_mcc: 0.2699
Epoch 5/1000
313/313 [==============================] - 11s 35ms/step - loss: 0.5001 - accuracy: 0.7546 - recall: 0.2389 - precision: 0.5577 - AUROC: 0.7420 - AUPRC: 0.4797 - f1_score: 0.3345 - balanced_accuracy: 0.5865 - specificity: 0.9341 - miss_rate: 0.7611 - fall_out: 0.0659 - mcc: 0.2414 - val_loss: 0.5224 - val_accuracy: 0.7491 - val_recall: 0.4894 - val_precision: 0.5147 - val_AUROC: 0.7593 - val_AUPRC: 0.5018 - val_f1_score: 0.5017 - val_balanced_accuracy: 0.6644 - val_specificity: 0.8394 - val_miss_rate: 0.5106 - val_fall_out: 0.1606 - val_mcc: 0.3344
Epoch 6/1000
313/313 [==============================] - 11s 35ms/step - loss: 0.4957 - accuracy: 0.7596 - recall: 0.2641 - precision: 0.5749 - AUROC: 0.7480 - AUPRC: 0.4991 - f1_score: 0.3620 - balanced_accuracy: 0.5981 - specificity: 0.9320 - miss_rate: 0.7359 - fall_out: 0.0680 - mcc: 0.2655 - val_loss: 0.4955 - val_accuracy: 0.7660 - val_recall: 0.3411 - val_precision: 0.5794 - val_AUROC: 0.7645 - val_AUPRC: 0.5217 - val_f1_score: 0.4294 - val_balanced_accuracy: 0.6275 - val_specificity: 0.9138 - val_miss_rate: 0.6589 - val_fall_out: 0.0862 - val_mcc: 0.3107
Epoch 7/1000
313/313 [==============================] - 11s 35ms/step - loss: 0.4922 - accuracy: 0.7617 - recall: 0.2797 - precision: 0.5796 - AUROC: 0.7532 - AUPRC: 0.5062 - f1_score: 0.3773 - balanced_accuracy: 0.6046 - specificity: 0.9294 - miss_rate: 0.7203 - fall_out: 0.0706 - mcc: 0.2771 - val_loss: 0.5192 - val_accuracy: 0.7519 - val_recall: 0.5631 - val_precision: 0.5178 - val_AUROC: 0.7663 - val_AUPRC: 0.5260 - val_f1_score: 0.5395 - val_balanced_accuracy: 0.6903 - val_specificity: 0.8175 - val_miss_rate: 0.4369 - val_fall_out: 0.1825 - val_mcc: 0.3707
Epoch 8/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.4870 - accuracy: 0.7673 - recall: 0.3119 - precision: 0.5936 - AUROC: 0.7597 - AUPRC: 0.5208 - f1_score: 0.4089 - balanced_accuracy: 0.6188 - specificity: 0.9257 - miss_rate: 0.6881 - fall_out: 0.0743 - mcc: 0.3037 - val_loss: 0.4939 - val_accuracy: 0.7659 - val_recall: 0.4128 - val_precision: 0.5635 - val_AUROC: 0.7672 - val_AUPRC: 0.5294 - val_f1_score: 0.4766 - val_balanced_accuracy: 0.6508 - val_specificity: 0.8887 - val_miss_rate: 0.5872 - val_fall_out: 0.1113 - val_mcc: 0.3370
Epoch 9/1000
313/313 [==============================] - 11s 35ms/step - loss: 0.4845 - accuracy: 0.7701 - recall: 0.3255 - precision: 0.6010 - AUROC: 0.7629 - AUPRC: 0.5273 - f1_score: 0.4223 - balanced_accuracy: 0.6251 - specificity: 0.9248 - miss_rate: 0.6745 - fall_out: 0.0752 - mcc: 0.3159 - val_loss: 0.4921 - val_accuracy: 0.7713 - val_recall: 0.4375 - val_precision: 0.5749 - val_AUROC: 0.7735 - val_AUPRC: 0.5445 - val_f1_score: 0.4969 - val_balanced_accuracy: 0.6625 - val_specificity: 0.8874 - val_miss_rate: 0.5625 - val_fall_out: 0.1126 - val_mcc: 0.3579
Epoch 10/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.4817 - accuracy: 0.7735 - recall: 0.3384 - precision: 0.6106 - AUROC: 0.7660 - AUPRC: 0.5363 - f1_score: 0.4355 - balanced_accuracy: 0.6317 - specificity: 0.9249 - miss_rate: 0.6616 - fall_out: 0.0751 - mcc: 0.3291 - val_loss: 0.4876 - val_accuracy: 0.7706 - val_recall: 0.4582 - val_precision: 0.5691 - val_AUROC: 0.7749 - val_AUPRC: 0.5462 - val_f1_score: 0.5077 - val_balanced_accuracy: 0.6687 - val_specificity: 0.8793 - val_miss_rate: 0.5418 - val_fall_out: 0.1207 - val_mcc: 0.3640
Epoch 11/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.4791 - accuracy: 0.7747 - recall: 0.3497 - precision: 0.6109 - AUROC: 0.7694 - AUPRC: 0.5423 - f1_score: 0.4448 - balanced_accuracy: 0.6361 - specificity: 0.9225 - miss_rate: 0.6503 - fall_out: 0.0775 - mcc: 0.3357 - val_loss: 0.4751 - val_accuracy: 0.7766 - val_recall: 0.3267 - val_precision: 0.6297 - val_AUROC: 0.7775 - val_AUPRC: 0.5528 - val_f1_score: 0.4302 - val_balanced_accuracy: 0.6299 - val_specificity: 0.9331 - val_miss_rate: 0.6733 - val_fall_out: 0.0669 - val_mcc: 0.3339
Epoch 12/1000
313/313 [==============================] - 11s 35ms/step - loss: 0.4765 - accuracy: 0.7784 - recall: 0.3537 - precision: 0.6248 - AUROC: 0.7722 - AUPRC: 0.5525 - f1_score: 0.4517 - balanced_accuracy: 0.6399 - specificity: 0.9261 - miss_rate: 0.6463 - fall_out: 0.0739 - mcc: 0.3467 - val_loss: 0.5003 - val_accuracy: 0.7637 - val_recall: 0.5461 - val_precision: 0.5420 - val_AUROC: 0.7768 - val_AUPRC: 0.5510 - val_f1_score: 0.5440 - val_balanced_accuracy: 0.6927 - val_specificity: 0.8394 - val_miss_rate: 0.4539 - val_fall_out: 0.1606 - val_mcc: 0.3845
Epoch 13/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.4733 - accuracy: 0.7809 - recall: 0.3643 - precision: 0.6311 - AUROC: 0.7756 - AUPRC: 0.5604 - f1_score: 0.4620 - balanced_accuracy: 0.6451 - specificity: 0.9259 - miss_rate: 0.6357 - fall_out: 0.0741 - mcc: 0.3567 - val_loss: 0.4834 - val_accuracy: 0.7756 - val_recall: 0.4398 - val_precision: 0.5874 - val_AUROC: 0.7770 - val_AUPRC: 0.5536 - val_f1_score: 0.5030 - val_balanced_accuracy: 0.6662 - val_specificity: 0.8925 - val_miss_rate: 0.5602 - val_fall_out: 0.1075 - val_mcc: 0.3683
313/313 [==============================] - 5s 17ms/step - loss: 0.4710 - accuracy: 0.7911 - recall: 0.4673 - precision: 0.6281 - AUROC: 0.7931 - AUPRC: 0.5981 - f1_score: 0.5359 - balanced_accuracy: 0.6855 - specificity: 0.9037 - miss_rate: 0.5327 - fall_out: 0.0963 - mcc: 0.4122
79/79 [==============================] - 2s 25ms/step - loss: 0.4834 - accuracy: 0.7756 - recall: 0.4398 - precision: 0.5874 - AUROC: 0.7770 - AUPRC: 0.5536 - f1_score: 0.5030 - balanced_accuracy: 0.6662 - specificity: 0.8925 - miss_rate: 0.5602 - fall_out: 0.1075 - mcc: 0.3683
-- HOLDOUT 7 --
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_6\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense (Dense) (None, 16) 16400
dropout_1 (Dropout) (None, 16) 0
dense_1 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_6\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 4, 'filters_0': 160, 'kernel_size_0': 16, 'conv_dropout_0': 0.2, 'dense_depth': 4, 'units_0': 128, 'dense_dropout_0': 0.4, 'units_1': 128, 'dense_dropout_1': 0.3, 'units_2': 32, 'dense_dropout_2': 0.4, 'filters_1': 32, 'kernel_size_1': 6, 'conv_dropout_1': 0.0, 'units_3': 16, 'dense_dropout_3': 0.3, 'filters_2': 72, 'kernel_size_2': 6, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.0, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.1, 'filters_5': 32, 'kernel_size_5': 6, 'conv_dropout_5': 0.0, 'units_4': 16, 'dense_dropout_4': 0.4}
Model: "CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d_1 (Conv1D) (None, 256, 160) 10400
dropout_2 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 32) 30752
dropout_3 (Dropout) (None, 128, 32) 0
max_pooling1d_2 (MaxPooling (None, 64, 32) 0
1D)
conv1d_3 (Conv1D) (None, 64, 72) 13896
dropout_4 (Dropout) (None, 64, 72) 0
max_pooling1d_3 (MaxPooling (None, 32, 72) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 3464
dropout_5 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
flatten_1 (Flatten) (None, 128) 0
dense_2 (Dense) (None, 128) 16512
dropout_6 (Dropout) (None, 128) 0
dense_3 (Dense) (None, 128) 16512
dropout_7 (Dropout) (None, 128) 0
dense_4 (Dense) (None, 32) 4128
dropout_8 (Dropout) (None, 32) 0
dense_5 (Dense) (None, 16) 528
dropout_9 (Dropout) (None, 16) 0
dense_6 (Dense) (None, 1) 17
=================================================================
Total params: 96,209
Trainable params: 96,209
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 12s 33ms/step - loss: 0.5680 - accuracy: 0.7384 - recall: 0.0087 - precision: 0.2806 - AUROC: 0.5990 - AUPRC: 0.3104 - f1_score: 0.0168 - balanced_accuracy: 0.5005 - specificity: 0.9923 - miss_rate: 0.9913 - fall_out: 0.0077 - mcc: 0.0046 - val_loss: 0.5563 - val_accuracy: 0.7419 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.6926 - val_AUPRC: 0.3904 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 2/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.5346 - accuracy: 0.7420 - recall: 0.0034 - precision: 0.5379 - AUROC: 0.6837 - AUPRC: 0.3954 - f1_score: 0.0068 - balanced_accuracy: 0.5012 - specificity: 0.9990 - miss_rate: 0.9966 - fall_out: 0.0010 - mcc: 0.0260 - val_loss: 0.5075 - val_accuracy: 0.7419 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.7347 - val_AUPRC: 0.4481 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 3/1000
313/313 [==============================] - 11s 35ms/step - loss: 0.5182 - accuracy: 0.7428 - recall: 0.0685 - precision: 0.5136 - AUROC: 0.7168 - AUPRC: 0.4341 - f1_score: 0.1209 - balanced_accuracy: 0.5230 - specificity: 0.9774 - miss_rate: 0.9315 - fall_out: 0.0226 - mcc: 0.1103 - val_loss: 0.6027 - val_accuracy: 0.7405 - val_recall: 0.2496 - val_precision: 0.4948 - val_AUROC: 0.6982 - val_AUPRC: 0.4267 - val_f1_score: 0.3318 - val_balanced_accuracy: 0.5804 - val_specificity: 0.9113 - val_miss_rate: 0.7504 - val_fall_out: 0.0887 - val_mcc: 0.2092
Epoch 4/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.5064 - accuracy: 0.7481 - recall: 0.1609 - precision: 0.5407 - AUROC: 0.7360 - AUPRC: 0.4625 - f1_score: 0.2480 - balanced_accuracy: 0.5567 - specificity: 0.9525 - miss_rate: 0.8391 - fall_out: 0.0475 - mcc: 0.1863 - val_loss: 0.4892 - val_accuracy: 0.7604 - val_recall: 0.2224 - val_precision: 0.5962 - val_AUROC: 0.7610 - val_AUPRC: 0.5038 - val_f1_score: 0.3240 - val_balanced_accuracy: 0.5850 - val_specificity: 0.9476 - val_miss_rate: 0.7776 - val_fall_out: 0.0524 - val_mcc: 0.2522
Epoch 5/1000
313/313 [==============================] - 11s 33ms/step - loss: 0.4960 - accuracy: 0.7588 - recall: 0.2356 - precision: 0.5806 - AUROC: 0.7516 - AUPRC: 0.4955 - f1_score: 0.3352 - balanced_accuracy: 0.5882 - specificity: 0.9408 - miss_rate: 0.7644 - fall_out: 0.0592 - mcc: 0.2521 - val_loss: 0.4867 - val_accuracy: 0.7651 - val_recall: 0.3562 - val_precision: 0.5723 - val_AUROC: 0.7642 - val_AUPRC: 0.5101 - val_f1_score: 0.4391 - val_balanced_accuracy: 0.6318 - val_specificity: 0.9074 - val_miss_rate: 0.6438 - val_fall_out: 0.0926 - val_mcc: 0.3141
Epoch 6/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.4881 - accuracy: 0.7662 - recall: 0.2912 - precision: 0.5966 - AUROC: 0.7616 - AUPRC: 0.5211 - f1_score: 0.3914 - balanced_accuracy: 0.6114 - specificity: 0.9315 - miss_rate: 0.7088 - fall_out: 0.0685 - mcc: 0.2937 - val_loss: 0.4935 - val_accuracy: 0.7680 - val_recall: 0.2936 - val_precision: 0.6042 - val_AUROC: 0.7653 - val_AUPRC: 0.5233 - val_f1_score: 0.3951 - val_balanced_accuracy: 0.6133 - val_specificity: 0.9331 - val_miss_rate: 0.7064 - val_fall_out: 0.0669 - val_mcc: 0.2994
Epoch 7/1000
313/313 [==============================] - 11s 34ms/step - loss: 0.4767 - accuracy: 0.7756 - recall: 0.3286 - precision: 0.6238 - AUROC: 0.7762 - AUPRC: 0.5479 - f1_score: 0.4305 - balanced_accuracy: 0.6298 - specificity: 0.9311 - miss_rate: 0.6714 - fall_out: 0.0689 - mcc: 0.3315 - val_loss: 0.4878 - val_accuracy: 0.7699 - val_recall: 0.3729 - val_precision: 0.5852 - val_AUROC: 0.7706 - val_AUPRC: 0.5320 - val_f1_score: 0.4555 - val_balanced_accuracy: 0.6405 - val_specificity: 0.9080 - val_miss_rate: 0.6271 - val_fall_out: 0.0920 - val_mcc: 0.3316
313/313 [==============================] - 5s 15ms/step - loss: 0.4566 - accuracy: 0.8020 - recall: 0.4453 - precision: 0.6772 - AUROC: 0.8208 - AUPRC: 0.6309 - f1_score: 0.5373 - balanced_accuracy: 0.6857 - specificity: 0.9262 - miss_rate: 0.5547 - fall_out: 0.0738 - mcc: 0.4330
79/79 [==============================] - 2s 25ms/step - loss: 0.4878 - accuracy: 0.7699 - recall: 0.3729 - precision: 0.5852 - AUROC: 0.7706 - AUPRC: 0.5320 - f1_score: 0.4555 - balanced_accuracy: 0.6405 - specificity: 0.9080 - miss_rate: 0.6271 - fall_out: 0.0920 - mcc: 0.3316
-- HOLDOUT 8 --
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_7\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense (Dense) (None, 16) 16400
dropout_1 (Dropout) (None, 16) 0
dense_1 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_7\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 8, 'conv_dropout_0': 0.2, 'dense_depth': 1, 'units_0': 16, 'dense_dropout_0': 0.3, 'filters_1': 160, 'kernel_size_1': 16, 'conv_dropout_1': 0.2, 'filters_2': 8, 'kernel_size_2': 16, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.2, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.2, 'units_1': 16, 'dense_dropout_1': 0.3, 'units_2': 64, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.3, 'filters_5': 96, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
Model: "CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d_1 (Conv1D) (None, 256, 160) 5280
dropout_2 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 160) 409760
dropout_3 (Dropout) (None, 128, 160) 0
max_pooling1d_2 (MaxPooling (None, 64, 160) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 20488
dropout_4 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 392
dropout_5 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 8) 392
dropout_6 (Dropout) (None, 16, 8) 0
max_pooling1d_5 (MaxPooling (None, 8, 8) 0
1D)
conv1d_6 (Conv1D) (None, 8, 96) 4704
dropout_7 (Dropout) (None, 8, 96) 0
max_pooling1d_6 (MaxPooling (None, 4, 96) 0
1D)
flatten_1 (Flatten) (None, 384) 0
dense_2 (Dense) (None, 16) 6160
dropout_8 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 447,193
Trainable params: 447,193
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 14s 40ms/step - loss: 0.5526 - accuracy: 0.7406 - recall: 0.0024 - precision: 0.2438 - AUROC: 0.6303 - AUPRC: 0.3449 - f1_score: 0.0047 - balanced_accuracy: 0.4999 - specificity: 0.9974 - miss_rate: 0.9976 - fall_out: 0.0026 - mcc: -0.0016 - val_loss: 0.5673 - val_accuracy: 0.7419 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.7172 - val_AUPRC: 0.4216 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 2/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.5194 - accuracy: 0.7416 - recall: 0.0043 - precision: 0.4378 - AUROC: 0.7114 - AUPRC: 0.4236 - f1_score: 0.0085 - balanced_accuracy: 0.5012 - specificity: 0.9981 - miss_rate: 0.9957 - fall_out: 0.0019 - mcc: 0.0206 - val_loss: 0.5303 - val_accuracy: 0.7419 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.7382 - val_AUPRC: 0.4496 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 3/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.5084 - accuracy: 0.7482 - recall: 0.1408 - precision: 0.5478 - AUROC: 0.7312 - AUPRC: 0.4618 - f1_score: 0.2240 - balanced_accuracy: 0.5502 - specificity: 0.9596 - miss_rate: 0.8592 - fall_out: 0.0404 - mcc: 0.1764 - val_loss: 0.5092 - val_accuracy: 0.7592 - val_recall: 0.1968 - val_precision: 0.6031 - val_AUROC: 0.7546 - val_AUPRC: 0.5032 - val_f1_score: 0.2968 - val_balanced_accuracy: 0.5759 - val_specificity: 0.9549 - val_miss_rate: 0.8032 - val_fall_out: 0.0451 - val_mcc: 0.2391
Epoch 4/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.4982 - accuracy: 0.7606 - recall: 0.2607 - precision: 0.5806 - AUROC: 0.7465 - AUPRC: 0.4953 - f1_score: 0.3598 - balanced_accuracy: 0.5976 - specificity: 0.9345 - miss_rate: 0.7393 - fall_out: 0.0655 - mcc: 0.2668 - val_loss: 0.4943 - val_accuracy: 0.7585 - val_recall: 0.1156 - val_precision: 0.6930 - val_AUROC: 0.7637 - val_AUPRC: 0.5220 - val_f1_score: 0.1981 - val_balanced_accuracy: 0.5489 - val_specificity: 0.9822 - val_miss_rate: 0.8844 - val_fall_out: 0.0178 - val_mcc: 0.2108
Epoch 5/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.4899 - accuracy: 0.7690 - recall: 0.3026 - precision: 0.6051 - AUROC: 0.7569 - AUPRC: 0.5228 - f1_score: 0.4034 - balanced_accuracy: 0.6169 - specificity: 0.9313 - miss_rate: 0.6974 - fall_out: 0.0687 - mcc: 0.3053 - val_loss: 0.4949 - val_accuracy: 0.7653 - val_recall: 0.1549 - val_precision: 0.7077 - val_AUROC: 0.7673 - val_AUPRC: 0.5356 - val_f1_score: 0.2542 - val_balanced_accuracy: 0.5663 - val_specificity: 0.9777 - val_miss_rate: 0.8451 - val_fall_out: 0.0223 - val_mcc: 0.2514
Epoch 6/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.4786 - accuracy: 0.7771 - recall: 0.3517 - precision: 0.6203 - AUROC: 0.7712 - AUPRC: 0.5480 - f1_score: 0.4489 - balanced_accuracy: 0.6384 - specificity: 0.9251 - miss_rate: 0.6483 - fall_out: 0.0749 - mcc: 0.3427 - val_loss: 0.4829 - val_accuracy: 0.7789 - val_recall: 0.3008 - val_precision: 0.6566 - val_AUROC: 0.7756 - val_AUPRC: 0.5556 - val_f1_score: 0.4126 - val_balanced_accuracy: 0.6230 - val_specificity: 0.9453 - val_miss_rate: 0.6992 - val_fall_out: 0.0547 - val_mcc: 0.3335
Epoch 7/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.4680 - accuracy: 0.7857 - recall: 0.3968 - precision: 0.6362 - AUROC: 0.7826 - AUPRC: 0.5739 - f1_score: 0.4888 - balanced_accuracy: 0.6589 - specificity: 0.9211 - miss_rate: 0.6032 - fall_out: 0.0789 - mcc: 0.3785 - val_loss: 0.4825 - val_accuracy: 0.7803 - val_recall: 0.3469 - val_precision: 0.6369 - val_AUROC: 0.7803 - val_AUPRC: 0.5666 - val_f1_score: 0.4492 - val_balanced_accuracy: 0.6390 - val_specificity: 0.9312 - val_miss_rate: 0.6531 - val_fall_out: 0.0688 - val_mcc: 0.3501
Epoch 8/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.4501 - accuracy: 0.8007 - recall: 0.4483 - precision: 0.6705 - AUROC: 0.8010 - AUPRC: 0.6161 - f1_score: 0.5374 - balanced_accuracy: 0.6858 - specificity: 0.9233 - miss_rate: 0.5517 - fall_out: 0.0767 - mcc: 0.4304 - val_loss: 0.4757 - val_accuracy: 0.7817 - val_recall: 0.4099 - val_precision: 0.6160 - val_AUROC: 0.7817 - val_AUPRC: 0.5673 - val_f1_score: 0.4923 - val_balanced_accuracy: 0.6605 - val_specificity: 0.9111 - val_miss_rate: 0.5901 - val_fall_out: 0.0889 - val_mcc: 0.3724
Epoch 9/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.4239 - accuracy: 0.8173 - recall: 0.5025 - precision: 0.7051 - AUROC: 0.8257 - AUPRC: 0.6667 - f1_score: 0.5868 - balanced_accuracy: 0.7147 - specificity: 0.9269 - miss_rate: 0.4975 - fall_out: 0.0731 - mcc: 0.4850 - val_loss: 0.4709 - val_accuracy: 0.7796 - val_recall: 0.2988 - val_precision: 0.6622 - val_AUROC: 0.7832 - val_AUPRC: 0.5716 - val_f1_score: 0.4118 - val_balanced_accuracy: 0.6229 - val_specificity: 0.9470 - val_miss_rate: 0.7012 - val_fall_out: 0.0530 - val_mcc: 0.3353
Epoch 10/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.3994 - accuracy: 0.8334 - recall: 0.5475 - precision: 0.7395 - AUROC: 0.8473 - AUPRC: 0.7084 - f1_score: 0.6292 - balanced_accuracy: 0.7402 - specificity: 0.9329 - miss_rate: 0.4525 - fall_out: 0.0671 - mcc: 0.5347 - val_loss: 0.4687 - val_accuracy: 0.7839 - val_recall: 0.3678 - val_precision: 0.6420 - val_AUROC: 0.7813 - val_AUPRC: 0.5715 - val_f1_score: 0.4677 - val_balanced_accuracy: 0.6482 - val_specificity: 0.9286 - val_miss_rate: 0.6322 - val_fall_out: 0.0714 - val_mcc: 0.3654
Epoch 11/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.3712 - accuracy: 0.8469 - recall: 0.5909 - precision: 0.7624 - AUROC: 0.8704 - AUPRC: 0.7501 - f1_score: 0.6658 - balanced_accuracy: 0.7634 - specificity: 0.9359 - miss_rate: 0.4091 - fall_out: 0.0641 - mcc: 0.5763 - val_loss: 0.4709 - val_accuracy: 0.7833 - val_recall: 0.3401 - val_precision: 0.6542 - val_AUROC: 0.7772 - val_AUPRC: 0.5715 - val_f1_score: 0.4476 - val_balanced_accuracy: 0.6388 - val_specificity: 0.9374 - val_miss_rate: 0.6599 - val_fall_out: 0.0626 - val_mcc: 0.3563
Epoch 12/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.3468 - accuracy: 0.8589 - recall: 0.6219 - precision: 0.7869 - AUROC: 0.8890 - AUPRC: 0.7823 - f1_score: 0.6947 - balanced_accuracy: 0.7816 - specificity: 0.9414 - miss_rate: 0.3781 - fall_out: 0.0586 - mcc: 0.6117 - val_loss: 0.4772 - val_accuracy: 0.7799 - val_recall: 0.4857 - val_precision: 0.5894 - val_AUROC: 0.7776 - val_AUPRC: 0.5759 - val_f1_score: 0.5326 - val_balanced_accuracy: 0.6840 - val_specificity: 0.8823 - val_miss_rate: 0.5143 - val_fall_out: 0.1177 - val_mcc: 0.3935
313/313 [==============================] - 5s 15ms/step - loss: 0.2742 - accuracy: 0.9175 - recall: 0.7888 - precision: 0.8794 - AUROC: 0.9528 - AUPRC: 0.9100 - f1_score: 0.8316 - balanced_accuracy: 0.8755 - specificity: 0.9623 - miss_rate: 0.2112 - fall_out: 0.0377 - mcc: 0.7792
79/79 [==============================] - 2s 24ms/step - loss: 0.4772 - accuracy: 0.7799 - recall: 0.4857 - precision: 0.5894 - AUROC: 0.7776 - AUPRC: 0.5759 - f1_score: 0.5326 - balanced_accuracy: 0.6840 - specificity: 0.8823 - miss_rate: 0.5143 - fall_out: 0.1177 - mcc: 0.3935
-- HOLDOUT 9 --
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_8\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense (Dense) (None, 16) 16400
dropout_1 (Dropout) (None, 16) 0
dense_1 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_8\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 4, 'filters_0': 160, 'kernel_size_0': 16, 'conv_dropout_0': 0.2, 'dense_depth': 3, 'units_0': 128, 'dense_dropout_0': 0.3, 'filters_1': 56, 'kernel_size_1': 8, 'conv_dropout_1': 0.0, 'filters_2': 56, 'kernel_size_2': 6, 'conv_dropout_2': 0.0, 'units_1': 32, 'dense_dropout_1': 0.4, 'units_2': 16, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.3, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.0, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.0, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0, 'units_4': 16, 'dense_dropout_4': 0.3}
Model: "CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d_1 (Conv1D) (None, 256, 160) 10400
dropout_2 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 56) 71736
dropout_3 (Dropout) (None, 128, 56) 0
max_pooling1d_2 (MaxPooling (None, 64, 56) 0
1D)
conv1d_3 (Conv1D) (None, 64, 56) 18872
dropout_4 (Dropout) (None, 64, 56) 0
max_pooling1d_3 (MaxPooling (None, 32, 56) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 2696
dropout_5 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
flatten_1 (Flatten) (None, 128) 0
dense_2 (Dense) (None, 128) 16512
dropout_6 (Dropout) (None, 128) 0
dense_3 (Dense) (None, 32) 4128
dropout_7 (Dropout) (None, 32) 0
dense_4 (Dense) (None, 16) 528
dropout_8 (Dropout) (None, 16) 0
dense_5 (Dense) (None, 1) 17
=================================================================
Total params: 124,889
Trainable params: 124,889
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 12s 33ms/step - loss: 0.5641 - accuracy: 0.7367 - recall: 0.0091 - precision: 0.2386 - AUROC: 0.6092 - AUPRC: 0.3196 - f1_score: 0.0176 - balanced_accuracy: 0.4995 - specificity: 0.9899 - miss_rate: 0.9909 - fall_out: 0.0101 - mcc: -0.0045 - val_loss: 0.5215 - val_accuracy: 0.7419 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.7102 - val_AUPRC: 0.4158 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 2/1000
313/313 [==============================] - 11s 33ms/step - loss: 0.5278 - accuracy: 0.7419 - recall: 0.0044 - precision: 0.5000 - AUROC: 0.6991 - AUPRC: 0.4114 - f1_score: 0.0087 - balanced_accuracy: 0.5014 - specificity: 0.9985 - miss_rate: 0.9956 - fall_out: 0.0015 - mcc: 0.0264 - val_loss: 0.5030 - val_accuracy: 0.7418 - val_recall: 3.8782e-04 - val_precision: 0.4000 - val_AUROC: 0.7362 - val_AUPRC: 0.4469 - val_f1_score: 7.7489e-04 - val_balanced_accuracy: 0.5001 - val_specificity: 0.9998 - val_miss_rate: 0.9996 - val_fall_out: 2.0243e-04 - val_mcc: 0.0051
Epoch 3/1000
313/313 [==============================] - 11s 33ms/step - loss: 0.5112 - accuracy: 0.7465 - recall: 0.1209 - precision: 0.5397 - AUROC: 0.7285 - AUPRC: 0.4545 - f1_score: 0.1976 - balanced_accuracy: 0.5425 - specificity: 0.9641 - miss_rate: 0.8791 - fall_out: 0.0359 - mcc: 0.1594 - val_loss: 0.4919 - val_accuracy: 0.7603 - val_recall: 0.1997 - val_precision: 0.6087 - val_AUROC: 0.7569 - val_AUPRC: 0.5036 - val_f1_score: 0.3008 - val_balanced_accuracy: 0.5775 - val_specificity: 0.9553 - val_miss_rate: 0.8003 - val_fall_out: 0.0447 - val_mcc: 0.2437
Epoch 4/1000
313/313 [==============================] - 11s 33ms/step - loss: 0.4997 - accuracy: 0.7570 - recall: 0.2354 - precision: 0.5710 - AUROC: 0.7467 - AUPRC: 0.4891 - f1_score: 0.3334 - balanced_accuracy: 0.5869 - specificity: 0.9385 - miss_rate: 0.7646 - fall_out: 0.0615 - mcc: 0.2468 - val_loss: 0.5012 - val_accuracy: 0.7541 - val_recall: 0.1454 - val_precision: 0.5976 - val_AUROC: 0.7576 - val_AUPRC: 0.4991 - val_f1_score: 0.2339 - val_balanced_accuracy: 0.5557 - val_specificity: 0.9659 - val_miss_rate: 0.8546 - val_fall_out: 0.0341 - val_mcc: 0.2008
Epoch 5/1000
313/313 [==============================] - 11s 33ms/step - loss: 0.4864 - accuracy: 0.7678 - recall: 0.3110 - precision: 0.5961 - AUROC: 0.7641 - AUPRC: 0.5244 - f1_score: 0.4088 - balanced_accuracy: 0.6189 - specificity: 0.9267 - miss_rate: 0.6890 - fall_out: 0.0733 - mcc: 0.3047 - val_loss: 0.5009 - val_accuracy: 0.7583 - val_recall: 0.1090 - val_precision: 0.7069 - val_AUROC: 0.7632 - val_AUPRC: 0.5243 - val_f1_score: 0.1888 - val_balanced_accuracy: 0.5466 - val_specificity: 0.9843 - val_miss_rate: 0.8910 - val_fall_out: 0.0157 - val_mcc: 0.2088
313/313 [==============================] - 5s 14ms/step - loss: 0.4774 - accuracy: 0.7635 - recall: 0.1258 - precision: 0.7502 - AUROC: 0.7968 - AUPRC: 0.5777 - f1_score: 0.2155 - balanced_accuracy: 0.5556 - specificity: 0.9854 - miss_rate: 0.8742 - fall_out: 0.0146 - mcc: 0.2392
79/79 [==============================] - 2s 26ms/step - loss: 0.5009 - accuracy: 0.7583 - recall: 0.1090 - precision: 0.7069 - AUROC: 0.7632 - AUPRC: 0.5243 - f1_score: 0.1888 - balanced_accuracy: 0.5466 - specificity: 0.9843 - miss_rate: 0.8910 - fall_out: 0.0157 - mcc: 0.2088
-- HOLDOUT 10 --
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_9\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense (Dense) (None, 16) 16400
dropout_1 (Dropout) (None, 16) 0
dense_1 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_9\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 3, 'filters_0': 144, 'kernel_size_0': 6, 'conv_dropout_0': 0.0, 'dense_depth': 1, 'units_0': 16, 'dense_dropout_0': 0.5, 'filters_1': 32, 'kernel_size_1': 8, 'conv_dropout_1': 0.2, 'filters_2': 104, 'kernel_size_2': 8, 'conv_dropout_2': 0.2, 'units_1': 16, 'dense_dropout_1': 0.5, 'units_2': 16, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.3}
Model: "CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d_1 (Conv1D) (None, 256, 144) 3600
dropout_2 (Dropout) (None, 256, 144) 0
max_pooling1d_1 (MaxPooling (None, 128, 144) 0
1D)
conv1d_2 (Conv1D) (None, 128, 32) 36896
dropout_3 (Dropout) (None, 128, 32) 0
max_pooling1d_2 (MaxPooling (None, 64, 32) 0
1D)
conv1d_3 (Conv1D) (None, 64, 104) 26728
dropout_4 (Dropout) (None, 64, 104) 0
max_pooling1d_3 (MaxPooling (None, 32, 104) 0
1D)
flatten_1 (Flatten) (None, 3328) 0
dense_2 (Dense) (None, 16) 53264
dropout_5 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 120,505
Trainable params: 120,505
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
313/313 [==============================] - 10s 28ms/step - loss: 0.5670 - accuracy: 0.7408 - recall: 0.0032 - precision: 0.3018 - AUROC: 0.6020 - AUPRC: 0.3179 - f1_score: 0.0064 - balanced_accuracy: 0.5003 - specificity: 0.9974 - miss_rate: 0.9968 - fall_out: 0.0026 - mcc: 0.0053 - val_loss: 0.5231 - val_accuracy: 0.7419 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.7213 - val_AUPRC: 0.4256 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 2/1000
313/313 [==============================] - 9s 27ms/step - loss: 0.5328 - accuracy: 0.7419 - recall: 0.0000e+00 - precision: 0.0000e+00 - AUROC: 0.6906 - AUPRC: 0.3904 - f1_score: 0.0000e+00 - balanced_accuracy: 0.5000 - specificity: 1.0000 - miss_rate: 1.0000 - fall_out: 0.0000e+00 - mcc: 0.0000e+00 - val_loss: 0.5104 - val_accuracy: 0.7419 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.7431 - val_AUPRC: 0.4581 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 3/1000
313/313 [==============================] - 9s 27ms/step - loss: 0.5222 - accuracy: 0.7428 - recall: 0.0291 - precision: 0.5333 - AUROC: 0.7123 - AUPRC: 0.4254 - f1_score: 0.0552 - balanced_accuracy: 0.5101 - specificity: 0.9911 - miss_rate: 0.9709 - fall_out: 0.0089 - mcc: 0.0752 - val_loss: 0.5280 - val_accuracy: 0.7468 - val_recall: 0.0471 - val_precision: 0.6263 - val_AUROC: 0.7546 - val_AUPRC: 0.4901 - val_f1_score: 0.0876 - val_balanced_accuracy: 0.5187 - val_specificity: 0.9902 - val_miss_rate: 0.9529 - val_fall_out: 0.0098 - val_mcc: 0.1184
Epoch 4/1000
313/313 [==============================] - 9s 27ms/step - loss: 0.5127 - accuracy: 0.7479 - recall: 0.1032 - precision: 0.5643 - AUROC: 0.7282 - AUPRC: 0.4546 - f1_score: 0.1745 - balanced_accuracy: 0.5377 - specificity: 0.9723 - miss_rate: 0.8968 - fall_out: 0.0277 - mcc: 0.1557 - val_loss: 0.5057 - val_accuracy: 0.7537 - val_recall: 0.0721 - val_precision: 0.7337 - val_AUROC: 0.7658 - val_AUPRC: 0.5330 - val_f1_score: 0.1314 - val_balanced_accuracy: 0.5315 - val_specificity: 0.9909 - val_miss_rate: 0.9279 - val_fall_out: 0.0091 - val_mcc: 0.1754
Epoch 5/1000
313/313 [==============================] - 9s 27ms/step - loss: 0.5081 - accuracy: 0.7533 - recall: 0.1346 - precision: 0.5985 - AUROC: 0.7355 - AUPRC: 0.4747 - f1_score: 0.2198 - balanced_accuracy: 0.5516 - specificity: 0.9686 - miss_rate: 0.8654 - fall_out: 0.0314 - mcc: 0.1931 - val_loss: 0.4930 - val_accuracy: 0.7513 - val_recall: 0.0533 - val_precision: 0.7597 - val_AUROC: 0.7704 - val_AUPRC: 0.5424 - val_f1_score: 0.0997 - val_balanced_accuracy: 0.5237 - val_specificity: 0.9941 - val_miss_rate: 0.9467 - val_fall_out: 0.0059 - val_mcc: 0.1557
Epoch 6/1000
313/313 [==============================] - 9s 27ms/step - loss: 0.5019 - accuracy: 0.7580 - recall: 0.1752 - precision: 0.6086 - AUROC: 0.7454 - AUPRC: 0.4947 - f1_score: 0.2721 - balanced_accuracy: 0.5680 - specificity: 0.9608 - miss_rate: 0.8248 - fall_out: 0.0392 - mcc: 0.2269 - val_loss: 0.5051 - val_accuracy: 0.7750 - val_recall: 0.4200 - val_precision: 0.5902 - val_AUROC: 0.7745 - val_AUPRC: 0.5522 - val_f1_score: 0.4908 - val_balanced_accuracy: 0.6593 - val_specificity: 0.8985 - val_miss_rate: 0.5800 - val_fall_out: 0.1015 - val_mcc: 0.3600
Epoch 7/1000
313/313 [==============================] - 9s 27ms/step - loss: 0.4945 - accuracy: 0.7647 - recall: 0.2270 - precision: 0.6207 - AUROC: 0.7566 - AUPRC: 0.5142 - f1_score: 0.3324 - balanced_accuracy: 0.5894 - specificity: 0.9517 - miss_rate: 0.7730 - fall_out: 0.0483 - mcc: 0.2675 - val_loss: 0.4930 - val_accuracy: 0.7698 - val_recall: 0.2143 - val_precision: 0.6689 - val_AUROC: 0.7736 - val_AUPRC: 0.5494 - val_f1_score: 0.3246 - val_balanced_accuracy: 0.5887 - val_specificity: 0.9631 - val_miss_rate: 0.7857 - val_fall_out: 0.0369 - val_mcc: 0.2818
313/313 [==============================] - 5s 15ms/step - loss: 0.4815 - accuracy: 0.7766 - recall: 0.2232 - precision: 0.7156 - AUROC: 0.7979 - AUPRC: 0.5820 - f1_score: 0.3402 - balanced_accuracy: 0.5962 - specificity: 0.9691 - miss_rate: 0.7768 - fall_out: 0.0309 - mcc: 0.3093
79/79 [==============================] - 2s 26ms/step - loss: 0.4930 - accuracy: 0.7698 - recall: 0.2143 - precision: 0.6689 - AUROC: 0.7736 - AUPRC: 0.5494 - f1_score: 0.3246 - balanced_accuracy: 0.5887 - specificity: 0.9631 - miss_rate: 0.7857 - fall_out: 0.0369 - mcc: 0.2818
CNN_metrics_promoters
[{'train_evaluation': {'loss': 0.46463608741760254,
'accuracy': 0.7859807014465332,
'recall': 0.4177939295768738,
'precision': 0.6285193562507629,
'AUROC': 0.7905188202857971,
'AUPRC': 0.5814822912216187,
'f1_score': 0.5019367933273315,
'balanced_accuracy': 0.6659390926361084,
'specificity': 0.914084255695343,
'miss_rate': 0.5822060704231262,
'fall_out': 0.08591575175523758,
'mcc': 0.3852105140686035},
'test_evaluation': {'loss': 0.4851917624473572,
'accuracy': 0.7685838937759399,
'recall': 0.3847197890281677,
'precision': 0.577751874923706,
'AUROC': 0.7632117867469788,
'AUPRC': 0.5343385338783264,
'f1_score': 0.4618787169456482,
'balanced_accuracy': 0.643439531326294,
'specificity': 0.9021592736244202,
'miss_rate': 0.6152802109718323,
'fall_out': 0.09784075617790222,
'mcc': 0.33274704217910767}},
{'train_evaluation': {'loss': 0.21400186419487,
'accuracy': 0.9345089197158813,
'recall': 0.7837091088294983,
'precision': 0.9544166326522827,
'AUROC': 0.9714784026145935,
'AUPRC': 0.9460043907165527,
'f1_score': 0.8606799840927124,
'balanced_accuracy': 0.8853429555892944,
'specificity': 0.9869768619537354,
'miss_rate': 0.2162909060716629,
'fall_out': 0.013023161329329014,
'mcc': 0.8252004981040955},
'test_evaluation': {'loss': 0.5257087349891663,
'accuracy': 0.7703359127044678,
'recall': 0.3866589069366455,
'precision': 0.5832114815711975,
'AUROC': 0.7475268244743347,
'AUPRC': 0.5227552056312561,
'f1_score': 0.4650186598300934,
'balanced_accuracy': 0.645252525806427,
'specificity': 0.9038461446762085,
'miss_rate': 0.6133410930633545,
'fall_out': 0.09615384787321091,
'mcc': 0.3375381529331207}},
{'train_evaluation': {'loss': 0.42886224389076233,
'accuracy': 0.8097466826438904,
'recall': 0.493139386177063,
'precision': 0.68174809217453,
'AUROC': 0.826547384262085,
'AUPRC': 0.6575813889503479,
'f1_score': 0.5723047256469727,
'balanced_accuracy': 0.70652174949646,
'specificity': 0.9199041724205017,
'miss_rate': 0.506860613822937,
'fall_out': 0.0800958201289177,
'mcc': 0.4638386368751526},
'test_evaluation': {'loss': 0.4834370017051697,
'accuracy': 0.7706862688064575,
'recall': 0.4169090688228607,
'precision': 0.5773361921310425,
'AUROC': 0.7681062817573547,
'AUPRC': 0.5443229079246521,
'f1_score': 0.48417970538139343,
'balanced_accuracy': 0.6553506255149841,
'specificity': 0.8937921524047852,
'miss_rate': 0.5830909609794617,
'fall_out': 0.10620782524347305,
'mcc': 0.34913551807403564}},
{'train_evaluation': {'loss': 0.2514292001724243,
'accuracy': 0.9088656306266785,
'recall': 0.725527286529541,
'precision': 0.9022610783576965,
'AUROC': 0.9469707012176514,
'AUPRC': 0.9015213251113892,
'f1_score': 0.8042998909950256,
'balanced_accuracy': 0.8490910530090332,
'specificity': 0.9726547598838806,
'miss_rate': 0.274472713470459,
'fall_out': 0.027345266193151474,
'mcc': 0.7533404231071472},
'test_evaluation': {'loss': 0.5135295391082764,
'accuracy': 0.7778445482254028,
'recall': 0.4219507575035095,
'precision': 0.5989540219306946,
'AUROC': 0.7653919458389282,
'AUPRC': 0.5595729351043701,
'f1_score': 0.49510806798934937,
'balanced_accuracy': 0.6618188619613647,
'specificity': 0.9016869068145752,
'miss_rate': 0.5780492424964905,
'fall_out': 0.0983130931854248,
'mcc': 0.3671722710132599}},
{'train_evaluation': {'loss': 0.28866279125213623,
'accuracy': 0.8746370673179626,
'recall': 0.5703272819519043,
'precision': 0.910589873790741,
'AUROC': 0.9581208825111389,
'AUPRC': 0.8886750340461731,
'f1_score': 0.7013683915138245,
'balanced_accuracy': 0.7754215598106384,
'specificity': 0.9805158376693726,
'miss_rate': 0.4296727180480957,
'fall_out': 0.019484134390950203,
'mcc': 0.6547654271125793},
'test_evaluation': {'loss': 0.5128974318504333,
'accuracy': 0.7717375159263611,
'recall': 0.27671125531196594,
'precision': 0.6322551965713501,
'AUROC': 0.7530701756477356,
'AUPRC': 0.5271884799003601,
'f1_score': 0.38494738936424255,
'balanced_accuracy': 0.6103529334068298,
'specificity': 0.9439945816993713,
'miss_rate': 0.7232887148857117,
'fall_out': 0.05600539967417717,
'mcc': 0.30509766936302185}},
{'train_evaluation': {'loss': 0.47102808952331543,
'accuracy': 0.7910867929458618,
'recall': 0.4673454463481903,
'precision': 0.6281115412712097,
'AUROC': 0.7931213974952698,
'AUPRC': 0.5980900526046753,
'f1_score': 0.5359317064285278,
'balanced_accuracy': 0.6855359673500061,
'specificity': 0.9037264585494995,
'miss_rate': 0.5326545238494873,
'fall_out': 0.09627355635166168,
'mcc': 0.41222357749938965},
'test_evaluation': {'loss': 0.48339036107063293,
'accuracy': 0.7756419777870178,
'recall': 0.4397905766963959,
'precision': 0.5874125957489014,
'AUROC': 0.7769896984100342,
'AUPRC': 0.5535800457000732,
'f1_score': 0.5029940009117126,
'balanced_accuracy': 0.6661503314971924,
'specificity': 0.8925101161003113,
'miss_rate': 0.5602094531059265,
'fall_out': 0.10748987644910812,
'mcc': 0.3682776987552643}},
{'train_evaluation': {'loss': 0.4566095471382141,
'accuracy': 0.8020374178886414,
'recall': 0.4452848434448242,
'precision': 0.6772361993789673,
'AUROC': 0.8208022713661194,
'AUPRC': 0.630867600440979,
'f1_score': 0.5372959971427917,
'balanced_accuracy': 0.6857237815856934,
'specificity': 0.9261627197265625,
'miss_rate': 0.5547151565551758,
'fall_out': 0.0738372802734375,
'mcc': 0.4330136179924011},
'test_evaluation': {'loss': 0.48783111572265625,
'accuracy': 0.7698853611946106,
'recall': 0.37289121747016907,
'precision': 0.5852099657058716,
'AUROC': 0.7705649137496948,
'AUPRC': 0.5319769382476807,
'f1_score': 0.455525279045105,
'balanced_accuracy': 0.6404604315757751,
'specificity': 0.9080296754837036,
'miss_rate': 0.6271088123321533,
'fall_out': 0.09197030961513519,
'mcc': 0.3316131830215454}},
{'train_evaluation': {'loss': 0.27423372864723206,
'accuracy': 0.9175385236740112,
'recall': 0.7887515425682068,
'precision': 0.8793513774871826,
'AUROC': 0.9527559876441956,
'AUPRC': 0.9100393652915955,
'f1_score': 0.8315910696983337,
'balanced_accuracy': 0.8755495548248291,
'specificity': 0.9623475670814514,
'miss_rate': 0.2112484872341156,
'fall_out': 0.037652455270290375,
'mcc': 0.7792206406593323},
'test_evaluation': {'loss': 0.47719261050224304,
'accuracy': 0.7798968553543091,
'recall': 0.4857475161552429,
'precision': 0.589411735534668,
'AUROC': 0.7775739431381226,
'AUPRC': 0.5758844614028931,
'f1_score': 0.5325821042060852,
'balanced_accuracy': 0.6840006113052368,
'specificity': 0.8822537064552307,
'miss_rate': 0.5142524838447571,
'fall_out': 0.11774628609418869,
'mcc': 0.3935089111328125}},
{'train_evaluation': {'loss': 0.4774458706378937,
'accuracy': 0.7635412216186523,
'recall': 0.1258181780576706,
'precision': 0.7502168416976929,
'AUROC': 0.7967522740364075,
'AUPRC': 0.5777274966239929,
'f1_score': 0.215495765209198,
'balanced_accuracy': 0.5556215047836304,
'specificity': 0.9854248762130737,
'miss_rate': 0.8741818070411682,
'fall_out': 0.014575144276022911,
'mcc': 0.23920510709285736},
'test_evaluation': {'loss': 0.500937819480896,
'accuracy': 0.7583220601081848,
'recall': 0.10897808521986008,
'precision': 0.7069182395935059,
'AUROC': 0.7632282972335815,
'AUPRC': 0.5243266820907593,
'f1_score': 0.18884408473968506,
'balanced_accuracy': 0.5466280579566956,
'specificity': 0.9842780232429504,
'miss_rate': 0.8910219073295593,
'fall_out': 0.015721997246146202,
'mcc': 0.20877063274383545}},
{'train_evaluation': {'loss': 0.4815472364425659,
'accuracy': 0.7765944004058838,
'recall': 0.2231757640838623,
'precision': 0.7156405448913574,
'AUROC': 0.7978525757789612,
'AUPRC': 0.582033634185791,
'f1_score': 0.34024468064308167,
'balanced_accuracy': 0.5961608290672302,
'specificity': 0.9691458940505981,
'miss_rate': 0.7768242359161377,
'fall_out': 0.03085409663617611,
'mcc': 0.30934399366378784},
'test_evaluation': {'loss': 0.4929780960083008,
'accuracy': 0.7697852253913879,
'recall': 0.21427185833454132,
'precision': 0.6688861846923828,
'AUROC': 0.7735965847969055,
'AUPRC': 0.5494359731674194,
'f1_score': 0.3245704174041748,
'balanced_accuracy': 0.588681161403656,
'specificity': 0.9630904197692871,
'miss_rate': 0.7857281565666199,
'fall_out': 0.03690958023071289,
'mcc': 0.28181034326553345}}]
CNN_metrics_estimate = model_metrics_holdout_estimate_evaluation(CNN_metrics_promoters, number_of_splits)
print("-- PROMOTERS --")
print(f"CNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']} -- test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']} -- test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']} -- test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN - Train history:")
plot_train_history(CNN_history_promoters)
-- PROMOTERS -- CNN Metrics - 10-holdouts estimate: Accuracy : train - 0.8364537358283997 -- test - 0.771271961927414 AUROC : train - 0.865492069721222 -- test - 0.765926045179367 AUPRC : train - 0.7274022579193116 -- test - 0.5423382163047791 -------------------------------------------------------------------------------- CNN - Train history:
# Saving results
import json
# Saving metrics
file = open("results/CNN_metrics_promoters.txt", "w")
string = repr(CNN_metrics_promoters)
file.write("CNN_metrics_promoters = " + string + "\n")
file.close()
# Saving history
with open("results/CNN_history_promoters.json", 'w') as h:
for holdout in range(number_of_splits):
json.dump(CNN_history_promoters[holdout].history, h)
MLP_distinct_metrics_promoters = {}
MLP_distinct_metrics_promoters["accuracy_test"] = []
MLP_distinct_metrics_promoters["AUROC_test"] = []
MLP_distinct_metrics_promoters["AUPRC_test"] = []
CNN_distinct_metrics_promoters = {}
CNN_distinct_metrics_promoters["accuracy_test"] = []
CNN_distinct_metrics_promoters["AUROC_test"] = []
CNN_distinct_metrics_promoters["AUPRC_test"] = []
for holdout in MLP_metrics_promoters["boruta_False"]:
MLP_distinct_metrics_promoters["accuracy_test"].append(holdout['accuracy_test'])
MLP_distinct_metrics_promoters["AUROC_test"].append(holdout['AUROC_test'])
MLP_distinct_metrics_promoters["AUPRC_test"].append(holdout['AUPRC_test'])
for holdout in CNN_metrics_promoters:
CNN_distinct_metrics_promoters["accuracy_test"].append(holdout["test_evaluation"]['accuracy'])
CNN_distinct_metrics_promoters["AUROC_test"].append(holdout["test_evaluation"]['AUROC'])
CNN_distinct_metrics_promoters["AUPRC_test"].append(holdout["test_evaluation"]['AUPRC'])
print("Promoters - Wilcoxon test - MLP VS CNN")
print(f"-Accuracy: {wilcoxon(MLP_distinct_metrics_promoters['accuracy_test'], CNN_distinct_metrics_promoters['accuracy_test'])}")
print(f"-AUROC: {wilcoxon(MLP_distinct_metrics_promoters['AUROC_test'], CNN_distinct_metrics_promoters['AUROC_test'])}")
print(f"-AUPRC: {wilcoxon(MLP_distinct_metrics_promoters['AUPRC_test'], CNN_distinct_metrics_promoters['AUPRC_test'])}")
Promoters - Wilcoxon test - MLP VS CNN -Accuracy: WilcoxonResult(statistic=0.0, pvalue=0.001953125) -AUROC: WilcoxonResult(statistic=0.0, pvalue=0.001953125) -AUPRC: WilcoxonResult(statistic=0.0, pvalue=0.001953125)
print("---- ENHANCERS ----")
CNN_metrics_enhancers = []
CNN_history_enhancers = []
for holdout_number, (train_indices, test_indices) in enumerate(holdouts_generator.split(bed_enhancers, enhancers_labels)):
print(f"-- HOLDOUT {holdout_number+1} --")
# Train/Test data
x_train_enhancers, x_test_enhancers = bed_enhancers.iloc[train_indices], bed_enhancers.iloc[test_indices]
y_train_enhancers, y_test_enhancers = enhancers_labels.iloc[train_indices], enhancers_labels.iloc[test_indices]
## Hyperparameter tuning
# Generate holdouts
for holdout_number_tuning, (train_indices_tuning, val_indices_tuning) in enumerate(holdouts_generator_tuning.split(x_train_enhancers, y_train_enhancers)):
# Train/Validation data
x_train_enhancers_tuning, x_val_enhancers_tuning = x_train_enhancers.iloc[train_indices_tuning], x_train_enhancers.iloc[val_indices_tuning]
y_train_enhancers_tuning, y_val_enhancers_tuning = y_train_enhancers.iloc[train_indices_tuning], y_train_enhancers.iloc[val_indices_tuning]
hp = kt.HyperParameters()
best_hyperparameters = hyperparameter_tuning_CNN(
x_train_enhancers_tuning,
x_val_enhancers_tuning,
y_train_enhancers_tuning.values,
y_val_enhancers_tuning.values,
build_CNN_hypermodel,
name = "CNN_hypermodel_enhancers_" + str(holdout_number),
directory='CNN_hypermodel_enhancers',
max_trials = 15,
epochs = 50,
batch_size = 256
)
#Build CNN with best set of hyperparameters
CNN = build_CNN(best_hyperparameters)
print("- Training model:\n")
CNN_holdout_metrics, CNN_holdout_history = train_CNN(
CNN,
x_train_enhancers,
x_test_enhancers,
y_train_enhancers.values,
y_test_enhancers.values,
genome,
epochs,
batch_size
)
CNN_metrics_enhancers.append(CNN_holdout_metrics)
CNN_history_enhancers.append(CNN_holdout_history)
Trial 15 Complete [00h 00m 49s]
multi_objective: -1.7022864073514938
Best multi_objective So Far: -1.7153688371181488
Total elapsed time: 00h 14m 05s
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 3, 'filters_0': 96, 'kernel_size_0': 6, 'conv_dropout_0': 0.1, 'dense_depth': 1, 'units_0': 32, 'dense_dropout_0': 0.4, 'filters_1': 96, 'kernel_size_1': 12, 'conv_dropout_1': 0.2, 'filters_2': 88, 'kernel_size_2': 16, 'conv_dropout_2': 0.2, 'filters_3': 104, 'kernel_size_3': 8, 'conv_dropout_3': 0.0, 'units_1': 32, 'dense_dropout_1': 0.3, 'units_2': 96, 'dense_dropout_2': 0.4, 'units_3': 48, 'dense_dropout_3': 0.4, 'units_4': 48, 'dense_dropout_4': 0.4}
Model: "CNN"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 96) 2400
dropout (Dropout) (None, 256, 96) 0
max_pooling1d (MaxPooling1D (None, 128, 96) 0
)
conv1d_1 (Conv1D) (None, 128, 96) 110688
dropout_1 (Dropout) (None, 128, 96) 0
max_pooling1d_1 (MaxPooling (None, 64, 96) 0
1D)
conv1d_2 (Conv1D) (None, 64, 88) 135256
dropout_2 (Dropout) (None, 64, 88) 0
max_pooling1d_2 (MaxPooling (None, 32, 88) 0
1D)
flatten (Flatten) (None, 2816) 0
dense (Dense) (None, 32) 90144
dropout_3 (Dropout) (None, 32) 0
dense_1 (Dense) (None, 1) 33
=================================================================
Total params: 338,521
Trainable params: 338,521
Non-trainable params: 0
_________________________________________________________________
- Training model:
Epoch 1/1000
198/198 [==============================] - 8s 36ms/step - loss: 0.3687 - accuracy: 0.8856 - recall: 0.0026 - precision: 0.1852 - AUROC: 0.5573 - AUPRC: 0.1414 - f1_score: 0.0052 - balanced_accuracy: 0.5006 - specificity: 0.9985 - miss_rate: 0.9974 - fall_out: 0.0015 - mcc: 0.0091 - val_loss: 0.4414 - val_accuracy: 0.8829 - val_recall: 0.0641 - val_precision: 0.3983 - val_AUROC: 0.6151 - val_AUPRC: 0.2008 - val_f1_score: 0.1104 - val_balanced_accuracy: 0.5259 - val_specificity: 0.9876 - val_miss_rate: 0.9359 - val_fall_out: 0.0124 - val_mcc: 0.1225
Epoch 2/1000
198/198 [==============================] - 7s 33ms/step - loss: 0.3570 - accuracy: 0.8871 - recall: 0.0141 - precision: 0.6000 - AUROC: 0.5808 - AUPRC: 0.1724 - f1_score: 0.0276 - balanced_accuracy: 0.5065 - specificity: 0.9988 - miss_rate: 0.9859 - fall_out: 0.0012 - mcc: 0.0793 - val_loss: 0.3451 - val_accuracy: 0.8873 - val_recall: 0.0118 - val_precision: 0.6800 - val_AUROC: 0.6194 - val_AUPRC: 0.2063 - val_f1_score: 0.0233 - val_balanced_accuracy: 0.5056 - val_specificity: 0.9993 - val_miss_rate: 0.9882 - val_fall_out: 7.1289e-04 - val_mcc: 0.0795
Epoch 3/1000
198/198 [==============================] - 7s 33ms/step - loss: 0.3555 - accuracy: 0.8869 - recall: 0.0145 - precision: 0.5570 - AUROC: 0.5822 - AUPRC: 0.1769 - f1_score: 0.0282 - balanced_accuracy: 0.5065 - specificity: 0.9985 - miss_rate: 0.9855 - fall_out: 0.0015 - mcc: 0.0760 - val_loss: 0.3532 - val_accuracy: 0.8869 - val_recall: 0.0028 - val_precision: 0.8000 - val_AUROC: 0.6264 - val_AUPRC: 0.2090 - val_f1_score: 0.0056 - val_balanced_accuracy: 0.5013 - val_specificity: 0.9999 - val_miss_rate: 0.9972 - val_fall_out: 8.9111e-05 - val_mcc: 0.0431
Epoch 4/1000
198/198 [==============================] - 7s 33ms/step - loss: 0.3512 - accuracy: 0.8869 - recall: 0.0157 - precision: 0.5556 - AUROC: 0.5933 - AUPRC: 0.1856 - f1_score: 0.0305 - balanced_accuracy: 0.5070 - specificity: 0.9984 - miss_rate: 0.9843 - fall_out: 0.0016 - mcc: 0.0790 - val_loss: 0.3526 - val_accuracy: 0.8869 - val_recall: 0.0021 - val_precision: 1.0000 - val_AUROC: 0.6320 - val_AUPRC: 0.2128 - val_f1_score: 0.0042 - val_balanced_accuracy: 0.5010 - val_specificity: 1.0000 - val_miss_rate: 0.9979 - val_fall_out: 0.0000e+00 - val_mcc: 0.0431
198/198 [==============================] - 4s 19ms/step - loss: 0.3510 - accuracy: 0.8868 - recall: 0.0028 - precision: 0.8000 - AUROC: 0.6460 - AUPRC: 0.2203 - f1_score: 0.0056 - balanced_accuracy: 0.5013 - specificity: 0.9999 - miss_rate: 0.9972 - fall_out: 8.9115e-05 - mcc: 0.0430
50/50 [==============================] - 2s 32ms/step - loss: 0.3526 - accuracy: 0.8869 - recall: 0.0021 - precision: 1.0000 - AUROC: 0.6320 - AUPRC: 0.2128 - f1_score: 0.0042 - balanced_accuracy: 0.5010 - specificity: 1.0000 - miss_rate: 0.9979 - fall_out: 0.0000e+00 - mcc: 0.0431
CNN_metrics_enhancers
[{'train_evaluation': {'loss': 0.29501962661743164,
'accuracy': 0.8865845203399658,
'recall': 0.0,
'precision': 0.0,
'AUROC': 0.8374033570289612,
'AUPRC': 0.3552652597427368,
'f1_score': 0.0,
'balanced_accuracy': 0.5,
'specificity': 1.0,
'miss_rate': 1.0,
'fall_out': 0.0,
'mcc': 0.0},
'test_evaluation': {'loss': 0.34381547570228577,
'accuracy': 0.8866239786148071,
'recall': 0.0,
'precision': 0.0,
'AUROC': 0.6243546009063721,
'AUPRC': 0.2190447896718979,
'f1_score': 0.0,
'balanced_accuracy': 0.5,
'specificity': 1.0,
'miss_rate': 1.0,
'fall_out': 0.0,
'mcc': 0.0}},
{'train_evaluation': {'loss': 0.3482652008533478,
'accuracy': 0.8865845203399658,
'recall': 0.0,
'precision': 0.0,
'AUROC': 0.6315912008285522,
'AUPRC': 0.21136806905269623,
'f1_score': 0.0,
'balanced_accuracy': 0.5,
'specificity': 1.0,
'miss_rate': 1.0,
'fall_out': 0.0,
'mcc': 0.0},
'test_evaluation': {'loss': 0.34882986545562744,
'accuracy': 0.8866239786148071,
'recall': 0.0,
'precision': 0.0,
'AUROC': 0.6267750263214111,
'AUPRC': 0.20636820793151855,
'f1_score': 0.0,
'balanced_accuracy': 0.5,
'specificity': 1.0,
'miss_rate': 1.0,
'fall_out': 0.0,
'mcc': 0.0}},
{'train_evaluation': {'loss': 0.3357076942920685,
'accuracy': 0.8868017792701721,
'recall': 0.0019157087663188577,
'precision': 1.0,
'AUROC': 0.6797792315483093,
'AUPRC': 0.24555028975009918,
'f1_score': 0.00382409174926579,
'balanced_accuracy': 0.5009578466415405,
'specificity': 1.0,
'miss_rate': 0.998084306716919,
'fall_out': 0.0,
'mcc': 0.041216589510440826},
'test_evaluation': {'loss': 0.34275010228157043,
'accuracy': 0.8867819905281067,
'recall': 0.0013937281910330057,
'precision': 1.0,
'AUROC': 0.6261276602745056,
'AUPRC': 0.20659670233726501,
'f1_score': 0.0027835769578814507,
'balanced_accuracy': 0.5006968379020691,
'specificity': 1.0,
'miss_rate': 0.998606264591217,
'fall_out': 0.0,
'mcc': 0.035155486315488815}},
{'train_evaluation': {'loss': 0.33552688360214233,
'accuracy': 0.8874733448028564,
'recall': 0.012016719207167625,
'precision': 0.7419354915618896,
'AUROC': 0.6710219383239746,
'AUPRC': 0.23610171675682068,
'f1_score': 0.023650385439395905,
'balanced_accuracy': 0.5057410001754761,
'specificity': 0.99946528673172,
'miss_rate': 0.9879832863807678,
'fall_out': 0.000534687889739871,
'mcc': 0.08502920717000961},
'test_evaluation': {'loss': 0.3432753086090088,
'accuracy': 0.8870190382003784,
'recall': 0.005574912764132023,
'precision': 0.7272727489471436,
'AUROC': 0.623015820980072,
'AUPRC': 0.19766809046268463,
'f1_score': 0.01106500718742609,
'balanced_accuracy': 0.5026537775993347,
'specificity': 0.9997326731681824,
'miss_rate': 0.9944250583648682,
'fall_out': 0.00026733201229944825,
'mcc': 0.05710640177130699}},
{'train_evaluation': {'loss': 0.3460206091403961,
'accuracy': 0.8865845203399658,
'recall': 0.0,
'precision': 0.0,
'AUROC': 0.633650541305542,
'AUPRC': 0.21331903338432312,
'f1_score': 0.0,
'balanced_accuracy': 0.5,
'specificity': 1.0,
'miss_rate': 1.0,
'fall_out': 0.0,
'mcc': 0.0},
'test_evaluation': {'loss': 0.3461054265499115,
'accuracy': 0.8866239786148071,
'recall': 0.0,
'precision': 0.0,
'AUROC': 0.6304782629013062,
'AUPRC': 0.20994004607200623,
'f1_score': 0.0,
'balanced_accuracy': 0.5,
'specificity': 1.0,
'miss_rate': 1.0,
'fall_out': 0.0,
'mcc': 0.0}},
{'train_evaluation': {'loss': 0.33860400319099426,
'accuracy': 0.8866832852363586,
'recall': 0.0015673980815336108,
'precision': 0.692307710647583,
'AUROC': 0.6530582904815674,
'AUPRC': 0.2222176343202591,
'f1_score': 0.0031277150847017765,
'balanced_accuracy': 0.5007391571998596,
'specificity': 0.9999108910560608,
'miss_rate': 0.9984325766563416,
'fall_out': 8.91146482899785e-05,
'mcc': 0.02925724908709526},
'test_evaluation': {'loss': 0.3401547372341156,
'accuracy': 0.8868610262870789,
'recall': 0.0020905924029648304,
'precision': 1.0,
'AUROC': 0.6326388716697693,
'AUPRC': 0.21864190697669983,
'f1_score': 0.004172461573034525,
'balanced_accuracy': 0.501045286655426,
'specificity': 1.0,
'miss_rate': 0.997909426689148,
'fall_out': 0.0,
'mcc': 0.04305820167064667}},
{'train_evaluation': {'loss': 0.34822630882263184,
'accuracy': 0.8866832852363586,
'recall': 0.0010449320543557405,
'precision': 0.8571428656578064,
'AUROC': 0.6514089703559875,
'AUPRC': 0.2222440391778946,
'f1_score': 0.002087319502606988,
'balanced_accuracy': 0.5005113482475281,
'specificity': 0.999977707862854,
'miss_rate': 0.9989550709724426,
'fall_out': 2.2278662072494626e-05,
'mcc': 0.027580438181757927},
'test_evaluation': {'loss': 0.35026171803474426,
'accuracy': 0.8866239786148071,
'recall': 0.0006968640955165029,
'precision': 0.5,
'AUROC': 0.6228237152099609,
'AUPRC': 0.20988352596759796,
'f1_score': 0.0013917884789407253,
'balanced_accuracy': 0.5003038644790649,
'specificity': 0.9999108910560608,
'miss_rate': 0.9993031620979309,
'fall_out': 8.911067561712116e-05,
'mcc': 0.015330009162425995}},
{'train_evaluation': {'loss': 0.3421771824359894,
'accuracy': 0.8865845203399658,
'recall': 0.0,
'precision': 0.0,
'AUROC': 0.6328660249710083,
'AUPRC': 0.20689919590950012,
'f1_score': 0.0,
'balanced_accuracy': 0.5,
'specificity': 1.0,
'miss_rate': 1.0,
'fall_out': 0.0,
'mcc': 0.0},
'test_evaluation': {'loss': 0.34444788098335266,
'accuracy': 0.8866239786148071,
'recall': 0.0,
'precision': 0.0,
'AUROC': 0.6169770956039429,
'AUPRC': 0.1905747503042221,
'f1_score': 0.0,
'balanced_accuracy': 0.5,
'specificity': 1.0,
'miss_rate': 1.0,
'fall_out': 0.0,
'mcc': 0.0}},
{'train_evaluation': {'loss': 0.3374524712562561,
'accuracy': 0.8872165679931641,
'recall': 0.006617903243750334,
'precision': 0.8636363744735718,
'AUROC': 0.6587575674057007,
'AUPRC': 0.22961513698101044,
'f1_score': 0.0131351538002491,
'balanced_accuracy': 0.5032420754432678,
'specificity': 0.9998663067817688,
'miss_rate': 0.9933820962905884,
'fall_out': 0.00013367197243496776,
'mcc': 0.06977705657482147},
'test_evaluation': {'loss': 0.3420707881450653,
'accuracy': 0.8867819905281067,
'recall': 0.0027874563820660114,
'precision': 0.6666666865348816,
'AUROC': 0.6285524964332581,
'AUPRC': 0.20559722185134888,
'f1_score': 0.005551700014621019,
'balanced_accuracy': 0.5013046264648438,
'specificity': 0.9998217821121216,
'miss_rate': 0.9972125291824341,
'fall_out': 0.00017822135123424232,
'mcc': 0.03800460323691368}},
{'train_evaluation': {'loss': 0.3510017693042755,
'accuracy': 0.8868215084075928,
'recall': 0.002786485478281975,
'precision': 0.800000011920929,
'AUROC': 0.6459976434707642,
'AUPRC': 0.22027051448822021,
'f1_score': 0.005553627386689186,
'balanced_accuracy': 0.5013486742973328,
'specificity': 0.9999108910560608,
'miss_rate': 0.9972135424613953,
'fall_out': 8.91146482899785e-05,
'mcc': 0.04304305836558342},
'test_evaluation': {'loss': 0.3525995910167694,
'accuracy': 0.8868610262870789,
'recall': 0.0020905924029648304,
'precision': 1.0,
'AUROC': 0.6319887638092041,
'AUPRC': 0.21282458305358887,
'f1_score': 0.004172461573034525,
'balanced_accuracy': 0.501045286655426,
'specificity': 1.0,
'miss_rate': 0.997909426689148,
'fall_out': 0.0,
'mcc': 0.04305820167064667}}]
CNN_metrics_estimate = model_metrics_holdout_estimate_evaluation(CNN_metrics_enhancers, number_of_splits)
print("-- ENHANCERS --")
print(f"CNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {CNN_metrics_estimate['accuracy_train']} -- test - {CNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {CNN_metrics_estimate['AUROC_train']} -- test - {CNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {CNN_metrics_estimate['AUPRC_train']} -- test - {CNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("CNN - Train history:")
plot_train_history(CNN_history_enhancers)
-- ENHANCERS -- CNN Metrics - 10-holdouts estimate: Accuracy : train - 0.8868017852306366 -- test - 0.8867424964904785 AUROC : train - 0.6695534765720368 -- test - 0.6263732314109802 AUPRC : train - 0.23628508895635605 -- test - 0.207713982462883 -------------------------------------------------------------------------------- CNN - Train history:
# Saving results
import json
# Saving metrics
file = open("results/CNN_metrics_enhancers.txt", "w")
string = repr(CNN_metrics_enhancers)
file.write("CNN_metrics_enhancers = " + string + "\n")
file.close()
# Saving history
with open("results/CNN_history_enhancers.json", 'w') as h:
for holdout in range(number_of_splits):
json.dump(CNN_history_enhancers[holdout].history, h)
MLP_distinct_metrics_enhancers = {}
MLP_distinct_metrics_enhancers["accuracy_test"] = []
MLP_distinct_metrics_enhancers["AUROC_test"] = []
MLP_distinct_metrics_enhancers["AUPRC_test"] = []
CNN_distinct_metrics_enhancers = {}
CNN_distinct_metrics_enhancers["accuracy_test"] = []
CNN_distinct_metrics_enhancers["AUROC_test"] = []
CNN_distinct_metrics_enhancers["AUPRC_test"] = []
for holdout in MLP_metrics_enhancers["boruta_False"]:
MLP_distinct_metrics_enhancers["accuracy_test"].append(holdout['accuracy_test'])
MLP_distinct_metrics_enhancers["AUROC_test"].append(holdout['AUROC_test'])
MLP_distinct_metrics_enhancers["AUPRC_test"].append(holdout['AUPRC_test'])
for holdout in CNN_metrics_enhancers:
CNN_distinct_metrics_enhancers["accuracy_test"].append(holdout["test_evaluation"]['accuracy'])
CNN_distinct_metrics_enhancers["AUROC_test"].append(holdout["test_evaluation"]['AUROC'])
CNN_distinct_metrics_enhancers["AUPRC_test"].append(holdout["test_evaluation"]['AUPRC'])
print("Enhancers - Wilcoxon test - MLP VS CNN")
print(f"-Accuracy: {wilcoxon(MLP_distinct_metrics_enhancers['accuracy_test'], CNN_distinct_metrics_enhancers['accuracy_test'])}")
print(f"-AUROC: {wilcoxon(MLP_distinct_metrics_enhancers['AUROC_test'], CNN_distinct_metrics_enhancers['AUROC_test'])}")
print(f"-AUPRC: {wilcoxon(MLP_distinct_metrics_enhancers['AUPRC_test'], CNN_distinct_metrics_enhancers['AUPRC_test'])}")
Enhancers - Wilcoxon test - MLP VS CNN -Accuracy: WilcoxonResult(statistic=2.0, pvalue=0.005859375) -AUROC: WilcoxonResult(statistic=0.0, pvalue=0.001953125) -AUPRC: WilcoxonResult(statistic=0.0, pvalue=0.001953125)
A multi-modal neural network is a model capable of jointly represent and exploit the information of both data modalities (epigenomic data and genomic sequence data). To shape this network the layers of a FFNN and a CNN are concatenated, each receiving their specific input, the outputs are then combined into a final dense layer and output layer.
The MMNN is here composed in two different ways to inspect the possible impact on the performances:
To have a statistically sound estimate of an architecture performance, multiple models are built and trained, each with the same architecture, over different portions of the data (holdouts) and, the average performance of those, is considered as an estimate of the overall performance of the architecture.
Here are the functions to build the optimized models, build the fixed models, and to build and train the MMNN.
The following step constructs the Mixed Sequence to train MMNNs with epigenomic data and DNA sequence data.
def get_mmnn_sequence(
genome: Genome,
bed: pd.DataFrame,
x: np.ndarray,
y: np.ndarray,
batch_size: int = 256
) -> MixedSequence:
""" Returns the genomic sequence data and epigenomic data for the MMNN training.
Parameters
-------------------------
genome: Genome,
The genome from where to extract the genomic sequence.
bed: pd.DataFrame,
The BED file coordinates describing where to extract the sequences.
x: np.ndarray,
The vector from where to extract the epigenomic data.
y: np.ndarray,
The values the model should predict.
batch_size: int,
The size of the batches to generate.
Returns
--------------------------
MixedSequence: MixedSequence
MixedSequence object to train a MMNN."""
return MixedSequence(
x={
"sequence_data": BedSequence(
genome,
bed,
batch_size=batch_size,
),
"epigenomic_data": VectorSequence(
x,
batch_size
)
},
y=VectorSequence(
y,
batch_size=batch_size
)
)
# Optimized MLP
def build_optimized_MLP(
best_hyperparameters: dict,
input_shape: tuple
) -> tuple[tf.keras.Model, layers.Layer, layers.Layer]:
""" Returns the optimized MLP model for constructing the MMNN.
Parameters
------------------------
best_hyperparameters: dict
The best set of hyperparameters for the model.
input_shape: tuple
The shape of the input.
Returns
------------------------
tuple: tuple[tf.keras.Model, layers.Layer, layers.Layer]:
The optimized MLP model for the MMNN, the input and last hidden layers of the model."""
print("- Building Optimized MLP:\n")
input_epigenomic_data = layers.Input(shape=(input_shape[1]), name="epigenomic_data")
hidden = layers.Dense(units = best_hyperparameters['units_0'], activation="relu")(input_epigenomic_data)
hidden = layers.Dropout(best_hyperparameters['dropout_0'])(hidden)
for i in range(best_hyperparameters['depth']-1):
hidden = layers.Dense(units = best_hyperparameters['units_' + str(i+1)], activation="relu")(hidden)
hidden = layers.Dropout(best_hyperparameters['dropout_' + str(i+1)])(hidden)
last_hidden_MLP = hidden
output_MLP = layers.Dense(1, activation="sigmoid")(last_hidden_MLP)
MLP = tf.keras.Model(
inputs = input_epigenomic_data,
outputs = output_MLP
)
MLP.compile(
optimizer="nadam",
loss="binary_crossentropy",
metrics=get_standard_binary_metrics()
)
MLP.summary()
return MLP, input_epigenomic_data, last_hidden_MLP
# Fixed MLP
def build_fixed_MLP(
input_shape: tuple
) -> tuple[tf.keras.Model, layers.Layer, layers.Layer]:
""" Returns the fixed MLP model for constructing the MMNN.
Parameters
------------------------
input_shape: tuple
The shape of the input.
Returns
------------------------
tuple: tuple[tf.keras.Model, layers.Layer, layers.Layer]:
The fixed MLP model for the MMNN, the input and last hidden layers of the model."""
print("- Building Fixed MLP:\n")
input_epigenomic_data = layers.Input(shape=(input_shape[1]), name="epigenomic_data")
hidden = layers.Dense(128, activation="relu")(input_epigenomic_data)
hidden = layers.Dropout(0.5)(hidden)
for _ in range(2):
hidden = layers.Dense(64, activation="relu")(hidden)
hidden = layers.Dropout(0.4)(hidden)
last_hidden_MLP = hidden
output_MLP = layers.Dense(1, activation="sigmoid")(last_hidden_MLP)
MLP = tf.keras.Model(
inputs = input_epigenomic_data,
outputs = output_MLP
)
MLP.compile(
optimizer="nadam",
loss="binary_crossentropy",
metrics=get_standard_binary_metrics()
)
MLP.summary()
return MLP, input_epigenomic_data, last_hidden_MLP
# Optimized CNN
def build_optimized_CNN(
best_hyperparameters: dict,
window_size: int
) -> tuple[tf.keras.Model, layers.Layer, layers.Layer]:
""" Returns the optimized CNN model for constructing the MMNN.
Parameters
------------------------
best_hyperparameters: dict
The best set of hyperparameters for the model.
window_size: int
The size of the window.
Returns
------------------------
tuple: tuple[tf.keras.Model, layers.Layer, layers.Layer]:
The optimized CNN model for the MMNN, the input and last hidden layers of the model."""
print("- Building Optimized CNN:\n")
input_sequence_data = layers.Input(shape=(window_size, 4), name="sequence_data")
hidden = layers.Conv1D(
filters = best_hyperparameters['filters_0'],
kernel_size = best_hyperparameters['kernel_size_0'],
padding = 'same',
activation="relu"
)(input_sequence_data)
hidden = layers.Dropout(best_hyperparameters['conv_dropout_0'])(hidden)
hidden = layers.MaxPooling1D(2)(hidden)
for i in range(best_hyperparameters['conv_depth']-1):
hidden = layers.Conv1D(
filters = best_hyperparameters['filters_' + str(i+1)],
kernel_size = best_hyperparameters['kernel_size_' + str(i+1)],
padding='same',
activation='relu'
)(hidden)
hidden = layers.Dropout(best_hyperparameters['conv_dropout_' + str(i+1)])(hidden)
hidden = layers.MaxPooling1D(2)(hidden)
hidden = layers.Flatten()(hidden)
for i in range(best_hyperparameters['dense_depth']):
hidden = layers.Dense(units = best_hyperparameters['units_' + str(i)], activation='relu')(hidden)
hidden = layers.Dropout(best_hyperparameters['dense_dropout_' + str(i)])(hidden)
last_hidden_CNN = hidden
output_CNN = layers.Dense(1, activation="sigmoid")(last_hidden_CNN)
CNN = tf.keras.Model(
inputs = input_sequence_data,
outputs = output_CNN
)
CNN.compile(
optimizer="nadam",
loss="binary_crossentropy",
metrics=get_standard_binary_metrics()
)
CNN.summary()
return CNN, input_sequence_data, last_hidden_CNN
# Fixed CNN
def build_fixed_CNN(
window_size: int
) -> tuple[tf.keras.Model, layers.Layer, layers.Layer]:
""" Returns the fixed CNN model for constructing the MMNN.
Parameters
------------------------
window_size: int
The size of the window.
Returns
------------------------
tuple: tuple[tf.keras.Model, layers.Layer, layers.Layer]:
The fixed CNN model for the MMNN, the input and last hidden layers of the model."""
print("- Building Fixed CNN:\n")
input_sequence_data = layers.Input(shape=(window_size, 4), name="sequence_data")
hidden = layers.Conv1D(
160,
kernel_size=8,
activation="linear",
kernel_regularizer=tf.keras.regularizers.l1_l2(),
activity_regularizer=tf.keras.regularizers.l1_l2()
)(input_sequence_data)
hidden = layers.Dropout(0.2)(hidden)
hidden = layers.Conv1D(
64,
kernel_size=8,
padding="same",
activation="relu",
kernel_regularizer=tf.keras.regularizers.l1_l2(),
activity_regularizer=tf.keras.regularizers.l1_l2()
)(hidden)
hidden = layers.Dropout(0.2)(hidden)
hidden = layers.MaxPooling1D(2)(hidden)
hidden = layers.Conv1D(
32,
kernel_size=8,
padding="same",
activation="relu",
kernel_regularizer=tf.keras.regularizers.l1_l2(),
activity_regularizer=tf.keras.regularizers.l1_l2()
)(hidden)
hidden = layers.Dropout(0.2)(hidden)
hidden = layers.MaxPooling1D(2)(hidden)
hidden = layers.Conv1D(
16,
kernel_size=6,
padding="same",
activation="relu",
kernel_regularizer=tf.keras.regularizers.l1_l2(),
activity_regularizer=tf.keras.regularizers.l1_l2()
)(hidden)
hidden = layers.Dropout(0.2)(hidden)
hidden = layers.MaxPooling1D(2)(hidden)
for _ in range(2):
hidden = layers.Conv1D(8, kernel_size=6, padding="same", activation="relu")(hidden)
hidden = layers.Dropout(0.2)(hidden)
hidden = layers.MaxPooling1D(2)(hidden)
hidden = layers.Flatten()(hidden)
hidden = layers.Dense(128, activation="relu")(hidden)
last_hidden_CNN = layers.Dropout(0.3)(hidden)
output_CNN = layers.Dense(1, activation="sigmoid")(last_hidden_CNN)
CNN = tf.keras.Model(
inputs = input_sequence_data,
outputs = output_CNN
)
CNN.compile(
optimizer="nadam",
loss="binary_crossentropy",
metrics=get_standard_binary_metrics()
)
CNN.summary()
return CNN, input_sequence_data, last_hidden_CNN
def build_MMNN(
input_epigenomic_data: layers.Layer,
input_sequence_data: layers.Layer,
last_hidden_MLP: layers.Layer,
last_hidden_CNN: layers.Layer
) -> tf.keras.Model:
""" Returns the MMNN model obtained concatenating the MLP and the CNN.
Parameters
------------------------
input_epigenomic_data: layers.Layer
The epigenomic data (MLP input).
input_sequence_data: layers.Layer
The sequence data (CNN input).
last_hidden_MLP: layers.Layer
The last hidden layer of the MLP model.
last_hidden_CNN: layers.Layer
The last hidden layer of the CNN model.
Returns
------------------------
MMNN: Model
The MMNN model."""
print("- Building MMNN:\n")
concatenation_layer = layers.Concatenate()([
last_hidden_MLP,
last_hidden_CNN
])
last_hidden_MMNN = layers.Dense(64, activation='relu')(concatenation_layer)
output_MMNN = layers.Dense(1, activation='sigmoid')(last_hidden_MMNN)
MMNN = tf.keras.Model(
inputs = [input_epigenomic_data, input_sequence_data],
outputs = output_MMNN
)
MMNN.compile(
optimizer='nadam',
loss='binary_crossentropy',
metrics=get_standard_binary_metrics()
)
MMNN.summary()
return MMNN
def train_MMNN(
model: tf.keras.Model,
bed_train: pd.DataFrame,
bed_test: pd.DataFrame,
x_train: np.ndarray,
x_test: np.ndarray,
y_train: np.ndarray,
y_test: np.ndarray,
genome: Genome,
epochs: int,
batch_size: int
):
""" Train the MMNN model and return the performance metrics (accuracy, AUROC, AUPRC) for train and test
and the training history.
Parameters
------------------------
bed_train: pd.DataFrame
The sequence data for training.
bed_test: pd.DataFrame
The sequence data for testing.
x_train: np.ndarray
The epigenomic input data for training.
x_test: np.ndarray
The epigenomic input data for testing.
y_train: np.ndarray
The labels of the input data for training.
y_test: np.ndarray
The labels of the input data for testing.
genome: Genome,
The genome from where to extract the genomic sequence.
epochs: int
The number of times the learning algorithm works through the dataset.
batch_size: int
The number of samples to work through before updating the model.
Returns
------------------------
MMNN: Model
The performance metrics and history of the MMNN model."""
train_sequence = get_mmnn_sequence(genome, bed_train, x_train, y_train, batch_size)
test_sequence = get_mmnn_sequence(genome, bed_test, x_test, y_test, batch_size)
MMNN_history = model.fit(
train_sequence,
validation_data = test_sequence,
epochs = epochs,
batch_size = batch_size,
callbacks = [EarlyStopping("val_loss", patience=2)]
)
train_evaluation = model.evaluate(train_sequence, return_dict=True)
test_evaluation = model.evaluate(test_sequence, return_dict=True)
MMNN_metrics = {"train_evaluation": train_evaluation, "test_evaluation": test_evaluation}
return MMNN_metrics, MMNN_history
NaN imputation and robust scaling of epigenomic data.
n = 5
promoters_epigenomes = knn_imputation(promoters_epigenomes, n)
enhancers_epigenomes = knn_imputation(enhancers_epigenomes, n)
print(f"Promoters NaN values: {promoters_epigenomes.isna().values.sum()}")
print(f"Enhancers NaN values: {enhancers_epigenomes.isna().values.sum()}")
Promoters NaN values: 0 Enhancers NaN values: 0
promoters_epigenomes = robust_scaler(promoters_epigenomes)
enhancers_epigenomes = robust_scaler(enhancers_epigenomes)
Wrapping together epigenomic and sequence data for the training and evaluation of the MMNN.
mixed_sequence_MMNN_promoters = get_mmnn_sequence(genome, bed_promoters, promoters_epigenomes.values, promoters_labels.values, batch_size)
mixed_sequence_MMNN_enhancers = get_mmnn_sequence(genome, bed_enhancers, enhancers_epigenomes.values, enhancers_labels.values, batch_size)
mixed_sequence_MMNN_promoters[0]
({'sequence_data': array([[[0., 0., 0., 1.],
[0., 0., 1., 0.],
[0., 0., 0., 1.],
...,
[0., 1., 0., 0.],
[0., 0., 1., 0.],
[0., 1., 0., 0.]],
[[1., 0., 0., 0.],
[0., 1., 0., 0.],
[0., 0., 1., 0.],
...,
[0., 0., 1., 0.],
[0., 0., 0., 1.],
[1., 0., 0., 0.]],
[[0., 0., 1., 0.],
[0., 0., 0., 1.],
[0., 0., 0., 1.],
...,
[0., 1., 0., 0.],
[0., 1., 0., 0.],
[0., 0., 1., 0.]],
...,
[[0., 1., 0., 0.],
[0., 0., 1., 0.],
[0., 1., 0., 0.],
...,
[0., 1., 0., 0.],
[0., 0., 0., 1.],
[0., 0., 0., 1.]],
[[0., 0., 1., 0.],
[0., 0., 1., 0.],
[0., 0., 1., 0.],
...,
[0., 1., 0., 0.],
[1., 0., 0., 0.],
[0., 1., 0., 0.]],
[[0., 0., 0., 1.],
[0., 0., 0., 1.],
[0., 0., 0., 1.],
...,
[0., 1., 0., 0.],
[1., 0., 0., 0.],
[0., 1., 0., 0.]]]),
'epigenomic_data': array([[-0.60240964, -0.28571429, -0.21888412, ..., -0.43949045,
-0.19178082, -0.2 ],
[ 0.5060241 , -0.11520737, -0.1888412 , ..., -0.14649682,
-0.21917808, -0.45333333],
[ 0.61445783, -0.24884793, -0.0472103 , ..., -0.40127389,
1.21917808, -0.28 ],
...,
[ 3.31325301, 0.63133641, 0.73390558, ..., 2.64968153,
0.31506849, 0.28 ],
[-0.6746988 , -0.19354839, 0.1888412 , ..., -0.43949045,
-0.26027397, -0.06666667],
[ 0.10843373, 4.22119816, 2.2832618 , ..., -0.22929936,
5.36986301, 0.74666667]])},
array([[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[ True],
[ True],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[ True],
[False],
[False],
[False],
[False],
[False],
[ True],
[ True],
[False],
[ True],
[ True],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[ True],
[False],
[False],
[False],
[ True],
[ True],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[ True],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[ True],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[ True],
[ True],
[False],
[False],
[ True],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[ True],
[ True],
[False],
[ True],
[False],
[ True],
[ True],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[ True],
[False],
[False],
[False],
[ True],
[False],
[ True],
[ True],
[ True],
[False],
[ True],
[ True],
[False],
[False],
[False],
[False],
[False],
[ True],
[ True],
[ True],
[False],
[ True],
[ True],
[False],
[False],
[ True],
[ True],
[False],
[ True],
[False],
[ True],
[ True],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[False],
[ True],
[False],
[False],
[False],
[ True],
[ True],
[False],
[False],
[False],
[False],
[ True],
[False],
[ True],
[False],
[ True],
[False],
[False],
[False],
[False]]))
epochs = 1000
batch_size = 256
print("---- PROMOTERS ----")
MMNN_metrics_promoters = {}
MMNN_history_promoters = {}
MMNN_metrics_promoters["tuning_True"] = []
MMNN_metrics_promoters["tuning_False"] = []
MMNN_history_promoters["tuning_True"] = []
MMNN_history_promoters["tuning_False"] = []
for holdout_number, (train_indices, test_indices) in enumerate(holdouts_generator.split(bed_promoters, promoters_labels)):
print(f"-- HOLDOUT {holdout_number+1} --")
# Train/Test data
x_train_promoters, x_test_promoters = promoters_epigenomes.iloc[train_indices], promoters_epigenomes.iloc[test_indices]
x_train_promoters_bed, x_test_promoters_bed = bed_promoters.iloc[train_indices], bed_promoters.iloc[test_indices]
y_train_promoters, y_test_promoters = promoters_labels.iloc[train_indices], promoters_labels.iloc[test_indices]
correlation_test = False
for use_tuning in (False, True):
if use_tuning:
## Hyperparameter tuning
# Generate holdouts
for holdout_number_tuning, (train_indices_tuning, val_indices_tuning) in enumerate(holdouts_generator_tuning.split(x_train_promoters, y_train_promoters)):
# Train/Validation data
x_train_promoters_tuning, x_val_promoters_tuning = x_train_promoters.iloc[train_indices_tuning], x_train_promoters.iloc[val_indices_tuning]
x_train_promoters_bed_tuning, x_val_promoters_bed_tuning = x_train_promoters_bed.iloc[train_indices_tuning], x_train_promoters_bed.iloc[val_indices_tuning]
y_train_promoters_tuning, y_val_promoters_tuning = y_train_promoters.iloc[train_indices_tuning], y_train_promoters.iloc[val_indices_tuning]
hp = kt.HyperParameters()
best_hyperparameters_MLP = hyperparameter_tuning(
x_train_promoters_tuning.values,
x_val_promoters_tuning.values,
y_train_promoters_tuning.values,
y_val_promoters_tuning.values,
build_MLP_hypermodel,
name = "MLP_hypermodel_promoters_" + str(holdout_number),
directory='MLP_hypermodel_promoters',
max_trials = 8,
epochs = 50,
batch_size = 256
)
best_hyperparameters_CNN = hyperparameter_tuning_CNN(
x_train_promoters_bed_tuning,
x_val_promoters_bed_tuning,
y_train_promoters_tuning.values,
y_val_promoters_tuning.values,
build_CNN_hypermodel,
name = "CNN_hypermodel_promoters_" + str(holdout_number),
directory='CNN_hypermodel_promoters',
max_trials = 15,
epochs = 50,
batch_size = 256
)
if not correlation_test:
useless_features = []
## Remove uncorrelated features with the output (epigenomic data)
uncorrelated_features = uncorrelated_features_test(x_train_promoters, y_train_promoters)
for feature in (x_train_promoters.columns):
if feature in (uncorrelated_features):
x_train_promoters_subset = x_train_promoters.drop(columns=feature)
x_test_promoters_subset = x_test_promoters.drop(columns=feature)
useless_features.append(feature)
## Remove correlated features with eachother (epigenomic data)
correlated_features = correlated_features_test(x_train_promoters_subset)
for feature in (x_train_promoters_subset.columns):
if feature in (correlated_features):
x_train_promoters_subset = x_train_promoters_subset.drop(columns=feature)
x_test_promoters_subset = x_test_promoters_subset.drop(columns=feature)
useless_features.append(feature)
if correlation_test:
# Correlation tests already executed, removing useless features found
print(f"- Removing uncorrelated/correlated features: {useless_features}\n")
for feature in (x_train_promoters.columns):
if feature in (useless_features):
x_train_promoters_subset = x_train_promoters.drop(columns=feature)
x_test_promoters_subset = x_test_promoters.drop(columns=feature)
correlation_test = True
if not use_tuning:
# Build Fixed MLP, CNN
MLP, input_epigenomic_data, last_hidden_MLP = build_fixed_MLP(x_train_promoters_subset.shape)
CNN, input_sequence_data, last_hidden_CNN = build_fixed_CNN(window_size)
if use_tuning:
# Build MLP, CNN with best set of hyperparameters
MLP, input_epigenomic_data, last_hidden_MLP = build_optimized_MLP(best_hyperparameters_MLP, x_train_promoters_subset.shape)
CNN, input_sequence_data, last_hidden_CNN = build_optimized_CNN(best_hyperparameters_CNN, window_size)
# Build MMNN
MMNN = build_MMNN(
input_epigenomic_data,
input_sequence_data,
last_hidden_MLP,
last_hidden_CNN
)
print("- Training MMNN model:\n")
MMNN_holdout_metrics, MMNN_holdout_history = train_MMNN(
MMNN,
x_train_promoters_bed,
x_test_promoters_bed,
x_train_promoters_subset.values,
x_test_promoters_subset.values,
y_train_promoters.values,
y_test_promoters.values,
genome,
epochs,
batch_size
)
MMNN_metrics_promoters["tuning_" + str(use_tuning)].append(MMNN_holdout_metrics)
MMNN_history_promoters["tuning_" + str(use_tuning)].append(MMNN_holdout_history)
---- PROMOTERS ----
-- HOLDOUT 1 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 18s 51ms/step - loss: 11.3649 - accuracy: 0.8027 - recall: 0.5632 - precision: 0.6323 - AUROC: 0.8594 - AUPRC: 0.6231 - f1_score: 0.5957 - balanced_accuracy: 0.7246 - specificity: 0.8860 - miss_rate: 0.4368 - fall_out: 0.1140 - mcc: 0.4672 - val_loss: 2.1894 - val_accuracy: 0.8375 - val_recall: 0.7846 - val_precision: 0.6546 - val_AUROC: 0.8982 - val_AUPRC: 0.6987 - val_f1_score: 0.7137 - val_balanced_accuracy: 0.8203 - val_specificity: 0.8559 - val_miss_rate: 0.2154 - val_fall_out: 0.1441 - val_mcc: 0.6064
Epoch 2/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.9286 - accuracy: 0.8315 - recall: 0.7266 - precision: 0.6570 - AUROC: 0.8905 - AUPRC: 0.6769 - f1_score: 0.6901 - balanced_accuracy: 0.7973 - specificity: 0.8680 - miss_rate: 0.2734 - fall_out: 0.1320 - mcc: 0.5762 - val_loss: 1.6605 - val_accuracy: 0.8408 - val_recall: 0.7900 - val_precision: 0.6601 - val_AUROC: 0.9021 - val_AUPRC: 0.7123 - val_f1_score: 0.7192 - val_balanced_accuracy: 0.8242 - val_specificity: 0.8584 - val_miss_rate: 0.2100 - val_fall_out: 0.1416 - val_mcc: 0.6141
Epoch 3/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.7424 - accuracy: 0.8377 - recall: 0.7509 - precision: 0.6641 - AUROC: 0.8962 - AUPRC: 0.6878 - f1_score: 0.7048 - balanced_accuracy: 0.8094 - specificity: 0.8678 - miss_rate: 0.2491 - fall_out: 0.1322 - mcc: 0.5956 - val_loss: 1.8317 - val_accuracy: 0.8436 - val_recall: 0.7673 - val_precision: 0.6727 - val_AUROC: 0.9048 - val_AUPRC: 0.7193 - val_f1_score: 0.7169 - val_balanced_accuracy: 0.8187 - val_specificity: 0.8701 - val_miss_rate: 0.2327 - val_fall_out: 0.1299 - val_mcc: 0.6120
Epoch 4/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.6188 - accuracy: 0.8401 - recall: 0.7539 - precision: 0.6689 - AUROC: 0.8996 - AUPRC: 0.6954 - f1_score: 0.7088 - balanced_accuracy: 0.8120 - specificity: 0.8701 - miss_rate: 0.2461 - fall_out: 0.1299 - mcc: 0.6012 - val_loss: 1.4301 - val_accuracy: 0.8451 - val_recall: 0.7516 - val_precision: 0.6813 - val_AUROC: 0.9059 - val_AUPRC: 0.7248 - val_f1_score: 0.7147 - val_balanced_accuracy: 0.8146 - val_specificity: 0.8777 - val_miss_rate: 0.2484 - val_fall_out: 0.1223 - val_mcc: 0.6102
Epoch 5/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.5303 - accuracy: 0.8418 - recall: 0.7514 - precision: 0.6736 - AUROC: 0.9015 - AUPRC: 0.7002 - f1_score: 0.7104 - balanced_accuracy: 0.8124 - specificity: 0.8733 - miss_rate: 0.2486 - fall_out: 0.1267 - mcc: 0.6038 - val_loss: 1.6121 - val_accuracy: 0.8426 - val_recall: 0.7861 - val_precision: 0.6650 - val_AUROC: 0.9065 - val_AUPRC: 0.7250 - val_f1_score: 0.7205 - val_balanced_accuracy: 0.8242 - val_specificity: 0.8622 - val_miss_rate: 0.2139 - val_fall_out: 0.1378 - val_mcc: 0.6161
Epoch 6/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.4623 - accuracy: 0.8428 - recall: 0.7559 - precision: 0.6744 - AUROC: 0.9043 - AUPRC: 0.7078 - f1_score: 0.7128 - balanced_accuracy: 0.8145 - specificity: 0.8730 - miss_rate: 0.2441 - fall_out: 0.1270 - mcc: 0.6069 - val_loss: 1.3311 - val_accuracy: 0.8449 - val_recall: 0.7553 - val_precision: 0.6796 - val_AUROC: 0.9076 - val_AUPRC: 0.7290 - val_f1_score: 0.7155 - val_balanced_accuracy: 0.8157 - val_specificity: 0.8761 - val_miss_rate: 0.2447 - val_fall_out: 0.1239 - val_mcc: 0.6109
Epoch 7/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.4071 - accuracy: 0.8453 - recall: 0.7613 - precision: 0.6785 - AUROC: 0.9056 - AUPRC: 0.7089 - f1_score: 0.7175 - balanced_accuracy: 0.8179 - specificity: 0.8745 - miss_rate: 0.2387 - fall_out: 0.1255 - mcc: 0.6133 - val_loss: 1.4661 - val_accuracy: 0.8452 - val_recall: 0.7755 - val_precision: 0.6739 - val_AUROC: 0.9086 - val_AUPRC: 0.7318 - val_f1_score: 0.7211 - val_balanced_accuracy: 0.8224 - val_specificity: 0.8694 - val_miss_rate: 0.2245 - val_fall_out: 0.1306 - val_mcc: 0.6176
Epoch 8/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.3602 - accuracy: 0.8476 - recall: 0.7625 - precision: 0.6835 - AUROC: 0.9076 - AUPRC: 0.7148 - f1_score: 0.7208 - balanced_accuracy: 0.8198 - specificity: 0.8772 - miss_rate: 0.2375 - fall_out: 0.1228 - mcc: 0.6182 - val_loss: 1.2121 - val_accuracy: 0.8473 - val_recall: 0.7504 - val_precision: 0.6869 - val_AUROC: 0.9098 - val_AUPRC: 0.7367 - val_f1_score: 0.7173 - val_balanced_accuracy: 0.8157 - val_specificity: 0.8810 - val_miss_rate: 0.2496 - val_fall_out: 0.1190 - val_mcc: 0.6141
Epoch 9/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.3183 - accuracy: 0.8488 - recall: 0.7601 - precision: 0.6873 - AUROC: 0.9093 - AUPRC: 0.7180 - f1_score: 0.7219 - balanced_accuracy: 0.8199 - specificity: 0.8797 - miss_rate: 0.2399 - fall_out: 0.1203 - mcc: 0.6199 - val_loss: 1.4416 - val_accuracy: 0.8475 - val_recall: 0.7289 - val_precision: 0.6952 - val_AUROC: 0.9104 - val_AUPRC: 0.7382 - val_f1_score: 0.7117 - val_balanced_accuracy: 0.8089 - val_specificity: 0.8888 - val_miss_rate: 0.2711 - val_fall_out: 0.1112 - val_mcc: 0.6084
Epoch 10/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.2804 - accuracy: 0.8509 - recall: 0.7676 - precision: 0.6898 - AUROC: 0.9110 - AUPRC: 0.7258 - f1_score: 0.7266 - balanced_accuracy: 0.8238 - specificity: 0.8799 - miss_rate: 0.2324 - fall_out: 0.1201 - mcc: 0.6262 - val_loss: 1.1978 - val_accuracy: 0.8465 - val_recall: 0.7723 - val_precision: 0.6778 - val_AUROC: 0.9103 - val_AUPRC: 0.7355 - val_f1_score: 0.7220 - val_balanced_accuracy: 0.8223 - val_specificity: 0.8723 - val_miss_rate: 0.2277 - val_fall_out: 0.1277 - val_mcc: 0.6191
Epoch 11/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.2473 - accuracy: 0.8496 - recall: 0.7664 - precision: 0.6871 - AUROC: 0.9111 - AUPRC: 0.7248 - f1_score: 0.7246 - balanced_accuracy: 0.8225 - specificity: 0.8785 - miss_rate: 0.2336 - fall_out: 0.1215 - mcc: 0.6233 - val_loss: 1.3162 - val_accuracy: 0.8482 - val_recall: 0.7245 - val_precision: 0.6987 - val_AUROC: 0.9103 - val_AUPRC: 0.7362 - val_f1_score: 0.7113 - val_balanced_accuracy: 0.8079 - val_specificity: 0.8913 - val_miss_rate: 0.2755 - val_fall_out: 0.1087 - val_mcc: 0.6086
Epoch 12/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.2156 - accuracy: 0.8508 - recall: 0.7648 - precision: 0.6904 - AUROC: 0.9123 - AUPRC: 0.7311 - f1_score: 0.7257 - balanced_accuracy: 0.8227 - specificity: 0.8806 - miss_rate: 0.2352 - fall_out: 0.1194 - mcc: 0.6251 - val_loss: 1.1709 - val_accuracy: 0.8461 - val_recall: 0.7770 - val_precision: 0.6756 - val_AUROC: 0.9105 - val_AUPRC: 0.7377 - val_f1_score: 0.7228 - val_balanced_accuracy: 0.8236 - val_specificity: 0.8702 - val_miss_rate: 0.2230 - val_fall_out: 0.1298 - val_mcc: 0.6199
Epoch 13/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.1829 - accuracy: 0.8523 - recall: 0.7690 - precision: 0.6927 - AUROC: 0.9142 - AUPRC: 0.7334 - f1_score: 0.7288 - balanced_accuracy: 0.8251 - specificity: 0.8813 - miss_rate: 0.2310 - fall_out: 0.1187 - mcc: 0.6293 - val_loss: 1.2260 - val_accuracy: 0.8456 - val_recall: 0.7675 - val_precision: 0.6774 - val_AUROC: 0.9109 - val_AUPRC: 0.7378 - val_f1_score: 0.7196 - val_balanced_accuracy: 0.8202 - val_specificity: 0.8728 - val_miss_rate: 0.2325 - val_fall_out: 0.1272 - val_mcc: 0.6160
Epoch 14/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.1565 - accuracy: 0.8539 - recall: 0.7723 - precision: 0.6953 - AUROC: 0.9149 - AUPRC: 0.7337 - f1_score: 0.7318 - balanced_accuracy: 0.8273 - specificity: 0.8823 - miss_rate: 0.2277 - fall_out: 0.1177 - mcc: 0.6334 - val_loss: 1.1338 - val_accuracy: 0.8497 - val_recall: 0.7374 - val_precision: 0.6977 - val_AUROC: 0.9120 - val_AUPRC: 0.7427 - val_f1_score: 0.7170 - val_balanced_accuracy: 0.8131 - val_specificity: 0.8888 - val_miss_rate: 0.2626 - val_fall_out: 0.1112 - val_mcc: 0.6153
Epoch 15/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.1301 - accuracy: 0.8539 - recall: 0.7753 - precision: 0.6944 - AUROC: 0.9162 - AUPRC: 0.7389 - f1_score: 0.7327 - balanced_accuracy: 0.8283 - specificity: 0.8813 - miss_rate: 0.2247 - fall_out: 0.1187 - mcc: 0.6344 - val_loss: 1.1484 - val_accuracy: 0.8492 - val_recall: 0.7630 - val_precision: 0.6873 - val_AUROC: 0.9121 - val_AUPRC: 0.7414 - val_f1_score: 0.7232 - val_balanced_accuracy: 0.8211 - val_specificity: 0.8792 - val_miss_rate: 0.2370 - val_fall_out: 0.1208 - val_mcc: 0.6216
Epoch 16/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.1046 - accuracy: 0.8553 - recall: 0.7743 - precision: 0.6980 - AUROC: 0.9167 - AUPRC: 0.7392 - f1_score: 0.7341 - balanced_accuracy: 0.8288 - specificity: 0.8834 - miss_rate: 0.2257 - fall_out: 0.1166 - mcc: 0.6367 - val_loss: 1.0926 - val_accuracy: 0.8486 - val_recall: 0.7049 - val_precision: 0.7075 - val_AUROC: 0.9117 - val_AUPRC: 0.7416 - val_f1_score: 0.7062 - val_balanced_accuracy: 0.8017 - val_specificity: 0.8986 - val_miss_rate: 0.2951 - val_fall_out: 0.1014 - val_mcc: 0.6042
Epoch 17/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.0791 - accuracy: 0.8561 - recall: 0.7757 - precision: 0.6995 - AUROC: 0.9180 - AUPRC: 0.7446 - f1_score: 0.7356 - balanced_accuracy: 0.8299 - specificity: 0.8841 - miss_rate: 0.2243 - fall_out: 0.1159 - mcc: 0.6388 - val_loss: 1.0910 - val_accuracy: 0.8482 - val_recall: 0.7681 - val_precision: 0.6832 - val_AUROC: 0.9125 - val_AUPRC: 0.7425 - val_f1_score: 0.7231 - val_balanced_accuracy: 0.8221 - val_specificity: 0.8760 - val_miss_rate: 0.2319 - val_fall_out: 0.1240 - val_mcc: 0.6211
Epoch 18/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.0568 - accuracy: 0.8572 - recall: 0.7738 - precision: 0.7029 - AUROC: 0.9187 - AUPRC: 0.7453 - f1_score: 0.7366 - balanced_accuracy: 0.8300 - specificity: 0.8862 - miss_rate: 0.2262 - fall_out: 0.1138 - mcc: 0.6404 - val_loss: 1.0720 - val_accuracy: 0.8503 - val_recall: 0.7623 - val_precision: 0.6901 - val_AUROC: 0.9125 - val_AUPRC: 0.7442 - val_f1_score: 0.7244 - val_balanced_accuracy: 0.8216 - val_specificity: 0.8809 - val_miss_rate: 0.2377 - val_fall_out: 0.1191 - val_mcc: 0.6234
Epoch 19/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.0348 - accuracy: 0.8584 - recall: 0.7794 - precision: 0.7038 - AUROC: 0.9197 - AUPRC: 0.7466 - f1_score: 0.7397 - balanced_accuracy: 0.8327 - specificity: 0.8859 - miss_rate: 0.2206 - fall_out: 0.1141 - mcc: 0.6444 - val_loss: 1.0442 - val_accuracy: 0.8493 - val_recall: 0.7332 - val_precision: 0.6982 - val_AUROC: 0.9121 - val_AUPRC: 0.7445 - val_f1_score: 0.7153 - val_balanced_accuracy: 0.8115 - val_specificity: 0.8897 - val_miss_rate: 0.2668 - val_fall_out: 0.1103 - val_mcc: 0.6133
Epoch 20/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.0140 - accuracy: 0.8597 - recall: 0.7822 - precision: 0.7060 - AUROC: 0.9207 - AUPRC: 0.7512 - f1_score: 0.7421 - balanced_accuracy: 0.8344 - specificity: 0.8867 - miss_rate: 0.2178 - fall_out: 0.1133 - mcc: 0.6477 - val_loss: 1.0037 - val_accuracy: 0.8465 - val_recall: 0.7886 - val_precision: 0.6730 - val_AUROC: 0.9119 - val_AUPRC: 0.7423 - val_f1_score: 0.7262 - val_balanced_accuracy: 0.8277 - val_specificity: 0.8667 - val_miss_rate: 0.2114 - val_fall_out: 0.1333 - val_mcc: 0.6243
Epoch 21/1000
313/313 [==============================] - 17s 51ms/step - loss: 0.9937 - accuracy: 0.8604 - recall: 0.7884 - precision: 0.7055 - AUROC: 0.9215 - AUPRC: 0.7538 - f1_score: 0.7447 - balanced_accuracy: 0.8370 - specificity: 0.8855 - miss_rate: 0.2116 - fall_out: 0.1145 - mcc: 0.6510 - val_loss: 0.9374 - val_accuracy: 0.8504 - val_recall: 0.7526 - val_precision: 0.6938 - val_AUROC: 0.9110 - val_AUPRC: 0.7396 - val_f1_score: 0.7220 - val_balanced_accuracy: 0.8185 - val_specificity: 0.8844 - val_miss_rate: 0.2474 - val_fall_out: 0.1156 - val_mcc: 0.6208
Epoch 22/1000
313/313 [==============================] - 16s 50ms/step - loss: 0.9740 - accuracy: 0.8614 - recall: 0.7817 - precision: 0.7103 - AUROC: 0.9230 - AUPRC: 0.7557 - f1_score: 0.7443 - balanced_accuracy: 0.8354 - specificity: 0.8891 - miss_rate: 0.2183 - fall_out: 0.1109 - mcc: 0.6509 - val_loss: 0.9748 - val_accuracy: 0.8500 - val_recall: 0.7409 - val_precision: 0.6970 - val_AUROC: 0.9128 - val_AUPRC: 0.7443 - val_f1_score: 0.7183 - val_balanced_accuracy: 0.8144 - val_specificity: 0.8879 - val_miss_rate: 0.2591 - val_fall_out: 0.1121 - val_mcc: 0.6167
Epoch 23/1000
313/313 [==============================] - 16s 51ms/step - loss: 0.9585 - accuracy: 0.8615 - recall: 0.7834 - precision: 0.7100 - AUROC: 0.9226 - AUPRC: 0.7548 - f1_score: 0.7449 - balanced_accuracy: 0.8360 - specificity: 0.8886 - miss_rate: 0.2166 - fall_out: 0.1114 - mcc: 0.6516 - val_loss: 0.9798 - val_accuracy: 0.8497 - val_recall: 0.7727 - val_precision: 0.6853 - val_AUROC: 0.9128 - val_AUPRC: 0.7442 - val_f1_score: 0.7264 - val_balanced_accuracy: 0.8246 - val_specificity: 0.8765 - val_miss_rate: 0.2273 - val_fall_out: 0.1235 - val_mcc: 0.6255
313/313 [==============================] - 8s 24ms/step - loss: 0.9461 - accuracy: 0.8709 - recall: 0.8148 - precision: 0.7213 - AUROC: 0.9331 - AUPRC: 0.7831 - f1_score: 0.7652 - balanced_accuracy: 0.8526 - specificity: 0.8905 - miss_rate: 0.1852 - fall_out: 0.1095 - mcc: 0.6791
79/79 [==============================] - 4s 43ms/step - loss: 0.9798 - accuracy: 0.8497 - recall: 0.7727 - precision: 0.6853 - AUROC: 0.9128 - AUPRC: 0.7442 - f1_score: 0.7264 - balanced_accuracy: 0.8246 - specificity: 0.8765 - miss_rate: 0.2273 - fall_out: 0.1235 - mcc: 0.6255
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_0\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_0\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 72, 'dropout_0': 0.4, 'units_1': 8, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.5, 'units_3': 8, 'dropout_3': 0.3, 'units_4': 104, 'dropout_4': 0.4}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_0\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_0\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 6, 'conv_dropout_0': 0.2, 'dense_depth': 1, 'units_0': 48, 'dense_dropout_0': 0.5, 'filters_1': 8, 'kernel_size_1': 6, 'conv_dropout_1': 0.0, 'filters_2': 8, 'kernel_size_2': 6, 'conv_dropout_2': 0.0, 'units_1': 16, 'dense_dropout_1': 0.3, 'units_2': 32, 'dense_dropout_2': 0.4, 'units_3': 80, 'dense_dropout_3': 0.4, 'units_4': 32, 'dense_dropout_4': 0.4, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.0, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.0, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 72) 40464
dropout_3 (Dropout) (None, 72) 0
dense_5 (Dense) (None, 1) 73
=================================================================
Total params: 40,537
Trainable params: 40,537
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 4000
dropout_4 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 8) 7688
dropout_5 (Dropout) (None, 128, 8) 0
max_pooling1d_2 (MaxPooling (None, 64, 8) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 392
dropout_6 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 392
dropout_7 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 8) 392
dropout_8 (Dropout) (None, 16, 8) 0
max_pooling1d_5 (MaxPooling (None, 8, 8) 0
1D)
conv1d_6 (Conv1D) (None, 8, 8) 392
dropout_9 (Dropout) (None, 8, 8) 0
max_pooling1d_6 (MaxPooling (None, 4, 8) 0
1D)
flatten_1 (Flatten) (None, 32) 0
dense_6 (Dense) (None, 48) 1584
dropout_10 (Dropout) (None, 48) 0
dense_7 (Dense) (None, 1) 49
=================================================================
Total params: 14,889
Trainable params: 14,889
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 4000 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 8) 7688 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 8) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 8) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 392 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 392 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_7[0][0]']
conv1d_5 (Conv1D) (None, 16, 8) 392 ['max_pooling1d_4[0][0]']
dropout_8 (Dropout) (None, 16, 8) 0 ['conv1d_5[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 8) 0 ['dropout_8[0][0]']
conv1d_6 (Conv1D) (None, 8, 8) 392 ['max_pooling1d_5[0][0]']
dropout_9 (Dropout) (None, 8, 8) 0 ['conv1d_6[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 8) 0 ['dropout_9[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
flatten_1 (Flatten) (None, 32) 0 ['max_pooling1d_6[0][0]']
dense_4 (Dense) (None, 72) 40464 ['epigenomic_data[0][0]']
dense_6 (Dense) (None, 48) 1584 ['flatten_1[0][0]']
dropout_3 (Dropout) (None, 72) 0 ['dense_4[0][0]']
dropout_10 (Dropout) (None, 48) 0 ['dense_6[0][0]']
concatenate (Concatenate) (None, 120) 0 ['dropout_3[0][0]',
'dropout_10[0][0]']
dense_8 (Dense) (None, 64) 7744 ['concatenate[0][0]']
dense_9 (Dense) (None, 1) 65 ['dense_8[0][0]']
==================================================================================================
Total params: 63,113
Trainable params: 63,113
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 14s 39ms/step - loss: 0.3787 - accuracy: 0.8184 - recall: 0.6447 - precision: 0.6493 - AUROC: 0.8768 - AUPRC: 0.6514 - f1_score: 0.6470 - balanced_accuracy: 0.7618 - specificity: 0.8788 - miss_rate: 0.3553 - fall_out: 0.1212 - mcc: 0.5247 - val_loss: 0.3409 - val_accuracy: 0.8387 - val_recall: 0.7875 - val_precision: 0.6563 - val_AUROC: 0.9004 - val_AUPRC: 0.7075 - val_f1_score: 0.7159 - val_balanced_accuracy: 0.8220 - val_specificity: 0.8565 - val_miss_rate: 0.2125 - val_fall_out: 0.1435 - val_mcc: 0.6094
Epoch 2/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3438 - accuracy: 0.8365 - recall: 0.7257 - precision: 0.6690 - AUROC: 0.8973 - AUPRC: 0.6902 - f1_score: 0.6962 - balanced_accuracy: 0.8004 - specificity: 0.8751 - miss_rate: 0.2743 - fall_out: 0.1249 - mcc: 0.5855 - val_loss: 0.3364 - val_accuracy: 0.8414 - val_recall: 0.7435 - val_precision: 0.6750 - val_AUROC: 0.9032 - val_AUPRC: 0.7143 - val_f1_score: 0.7076 - val_balanced_accuracy: 0.8094 - val_specificity: 0.8754 - val_miss_rate: 0.2565 - val_fall_out: 0.1246 - val_mcc: 0.6004
Epoch 3/1000
313/313 [==============================] - 12s 39ms/step - loss: 0.3351 - accuracy: 0.8410 - recall: 0.7375 - precision: 0.6760 - AUROC: 0.9028 - AUPRC: 0.7087 - f1_score: 0.7054 - balanced_accuracy: 0.8072 - specificity: 0.8770 - miss_rate: 0.2625 - fall_out: 0.1230 - mcc: 0.5979 - val_loss: 0.3298 - val_accuracy: 0.8436 - val_recall: 0.7551 - val_precision: 0.6765 - val_AUROC: 0.9064 - val_AUPRC: 0.7279 - val_f1_score: 0.7136 - val_balanced_accuracy: 0.8147 - val_specificity: 0.8744 - val_miss_rate: 0.2449 - val_fall_out: 0.1256 - val_mcc: 0.6082
Epoch 4/1000
313/313 [==============================] - 15s 47ms/step - loss: 0.3285 - accuracy: 0.8445 - recall: 0.7463 - precision: 0.6816 - AUROC: 0.9066 - AUPRC: 0.7175 - f1_score: 0.7125 - balanced_accuracy: 0.8125 - specificity: 0.8787 - miss_rate: 0.2537 - fall_out: 0.1213 - mcc: 0.6074 - val_loss: 0.3299 - val_accuracy: 0.8435 - val_recall: 0.8045 - val_precision: 0.6620 - val_AUROC: 0.9078 - val_AUPRC: 0.7303 - val_f1_score: 0.7264 - val_balanced_accuracy: 0.8308 - val_specificity: 0.8571 - val_miss_rate: 0.1955 - val_fall_out: 0.1429 - val_mcc: 0.6240
Epoch 5/1000
313/313 [==============================] - 12s 38ms/step - loss: 0.3247 - accuracy: 0.8474 - recall: 0.7553 - precision: 0.6856 - AUROC: 0.9089 - AUPRC: 0.7218 - f1_score: 0.7188 - balanced_accuracy: 0.8174 - specificity: 0.8795 - miss_rate: 0.2447 - fall_out: 0.1205 - mcc: 0.6158 - val_loss: 0.3289 - val_accuracy: 0.8460 - val_recall: 0.7760 - val_precision: 0.6756 - val_AUROC: 0.9080 - val_AUPRC: 0.7314 - val_f1_score: 0.7223 - val_balanced_accuracy: 0.8232 - val_specificity: 0.8703 - val_miss_rate: 0.2240 - val_fall_out: 0.1297 - val_mcc: 0.6193
Epoch 6/1000
313/313 [==============================] - 12s 38ms/step - loss: 0.3214 - accuracy: 0.8490 - recall: 0.7592 - precision: 0.6881 - AUROC: 0.9111 - AUPRC: 0.7292 - f1_score: 0.7219 - balanced_accuracy: 0.8197 - specificity: 0.8803 - miss_rate: 0.2408 - fall_out: 0.1197 - mcc: 0.6200 - val_loss: 0.3274 - val_accuracy: 0.8467 - val_recall: 0.7421 - val_precision: 0.6883 - val_AUROC: 0.9096 - val_AUPRC: 0.7363 - val_f1_score: 0.7142 - val_balanced_accuracy: 0.8126 - val_specificity: 0.8831 - val_miss_rate: 0.2579 - val_fall_out: 0.1169 - val_mcc: 0.6104
Epoch 7/1000
313/313 [==============================] - 12s 38ms/step - loss: 0.3176 - accuracy: 0.8510 - recall: 0.7624 - precision: 0.6919 - AUROC: 0.9131 - AUPRC: 0.7355 - f1_score: 0.7254 - balanced_accuracy: 0.8221 - specificity: 0.8819 - miss_rate: 0.2376 - fall_out: 0.1181 - mcc: 0.6250 - val_loss: 0.3254 - val_accuracy: 0.8453 - val_recall: 0.7801 - val_precision: 0.6727 - val_AUROC: 0.9089 - val_AUPRC: 0.7287 - val_f1_score: 0.7225 - val_balanced_accuracy: 0.8240 - val_specificity: 0.8679 - val_miss_rate: 0.2199 - val_fall_out: 0.1321 - val_mcc: 0.6192
Epoch 8/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3139 - accuracy: 0.8532 - recall: 0.7660 - precision: 0.6959 - AUROC: 0.9154 - AUPRC: 0.7414 - f1_score: 0.7293 - balanced_accuracy: 0.8248 - specificity: 0.8835 - miss_rate: 0.2340 - fall_out: 0.1165 - mcc: 0.6302 - val_loss: 0.3220 - val_accuracy: 0.8478 - val_recall: 0.7737 - val_precision: 0.6805 - val_AUROC: 0.9114 - val_AUPRC: 0.7415 - val_f1_score: 0.7241 - val_balanced_accuracy: 0.8237 - val_specificity: 0.8736 - val_miss_rate: 0.2263 - val_fall_out: 0.1264 - val_mcc: 0.6221
Epoch 9/1000
313/313 [==============================] - 14s 43ms/step - loss: 0.3118 - accuracy: 0.8545 - recall: 0.7651 - precision: 0.6994 - AUROC: 0.9168 - AUPRC: 0.7467 - f1_score: 0.7308 - balanced_accuracy: 0.8254 - specificity: 0.8856 - miss_rate: 0.2349 - fall_out: 0.1144 - mcc: 0.6326 - val_loss: 0.3218 - val_accuracy: 0.8472 - val_recall: 0.7776 - val_precision: 0.6779 - val_AUROC: 0.9112 - val_AUPRC: 0.7406 - val_f1_score: 0.7243 - val_balanced_accuracy: 0.8245 - val_specificity: 0.8715 - val_miss_rate: 0.2224 - val_fall_out: 0.1285 - val_mcc: 0.6221
Epoch 10/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.3085 - accuracy: 0.8557 - recall: 0.7692 - precision: 0.7008 - AUROC: 0.9186 - AUPRC: 0.7521 - f1_score: 0.7334 - balanced_accuracy: 0.8275 - specificity: 0.8857 - miss_rate: 0.2308 - fall_out: 0.1143 - mcc: 0.6360 - val_loss: 0.3227 - val_accuracy: 0.8484 - val_recall: 0.7597 - val_precision: 0.6865 - val_AUROC: 0.9112 - val_AUPRC: 0.7412 - val_f1_score: 0.7213 - val_balanced_accuracy: 0.8195 - val_specificity: 0.8793 - val_miss_rate: 0.2403 - val_fall_out: 0.1207 - val_mcc: 0.6191
Epoch 11/1000
313/313 [==============================] - 13s 42ms/step - loss: 0.3050 - accuracy: 0.8576 - recall: 0.7664 - precision: 0.7068 - AUROC: 0.9206 - AUPRC: 0.7564 - f1_score: 0.7354 - balanced_accuracy: 0.8279 - specificity: 0.8894 - miss_rate: 0.2336 - fall_out: 0.1106 - mcc: 0.6392 - val_loss: 0.3212 - val_accuracy: 0.8502 - val_recall: 0.7450 - val_precision: 0.6960 - val_AUROC: 0.9117 - val_AUPRC: 0.7385 - val_f1_score: 0.7197 - val_balanced_accuracy: 0.8159 - val_specificity: 0.8868 - val_miss_rate: 0.2550 - val_fall_out: 0.1132 - val_mcc: 0.6183
Epoch 12/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.3018 - accuracy: 0.8588 - recall: 0.7748 - precision: 0.7064 - AUROC: 0.9222 - AUPRC: 0.7613 - f1_score: 0.7390 - balanced_accuracy: 0.8314 - specificity: 0.8880 - miss_rate: 0.2252 - fall_out: 0.1120 - mcc: 0.6438 - val_loss: 0.3217 - val_accuracy: 0.8485 - val_recall: 0.7564 - val_precision: 0.6878 - val_AUROC: 0.9116 - val_AUPRC: 0.7369 - val_f1_score: 0.7205 - val_balanced_accuracy: 0.8185 - val_specificity: 0.8805 - val_miss_rate: 0.2436 - val_fall_out: 0.1195 - val_mcc: 0.6182
Epoch 13/1000
313/313 [==============================] - 15s 47ms/step - loss: 0.2994 - accuracy: 0.8600 - recall: 0.7742 - precision: 0.7098 - AUROC: 0.9236 - AUPRC: 0.7672 - f1_score: 0.7406 - balanced_accuracy: 0.8320 - specificity: 0.8899 - miss_rate: 0.2258 - fall_out: 0.1101 - mcc: 0.6461 - val_loss: 0.3211 - val_accuracy: 0.8502 - val_recall: 0.7533 - val_precision: 0.6930 - val_AUROC: 0.9118 - val_AUPRC: 0.7415 - val_f1_score: 0.7219 - val_balanced_accuracy: 0.8186 - val_specificity: 0.8839 - val_miss_rate: 0.2467 - val_fall_out: 0.1161 - val_mcc: 0.6206
Epoch 14/1000
313/313 [==============================] - 12s 38ms/step - loss: 0.2965 - accuracy: 0.8634 - recall: 0.7817 - precision: 0.7155 - AUROC: 0.9254 - AUPRC: 0.7718 - f1_score: 0.7471 - balanced_accuracy: 0.8368 - specificity: 0.8918 - miss_rate: 0.2183 - fall_out: 0.1082 - mcc: 0.6550 - val_loss: 0.3212 - val_accuracy: 0.8516 - val_recall: 0.7564 - val_precision: 0.6955 - val_AUROC: 0.9117 - val_AUPRC: 0.7386 - val_f1_score: 0.7247 - val_balanced_accuracy: 0.8206 - val_specificity: 0.8848 - val_miss_rate: 0.2436 - val_fall_out: 0.1152 - val_mcc: 0.6244
Epoch 15/1000
313/313 [==============================] - 15s 47ms/step - loss: 0.2922 - accuracy: 0.8648 - recall: 0.7822 - precision: 0.7188 - AUROC: 0.9276 - AUPRC: 0.7804 - f1_score: 0.7492 - balanced_accuracy: 0.8379 - specificity: 0.8936 - miss_rate: 0.2178 - fall_out: 0.1064 - mcc: 0.6580 - val_loss: 0.3209 - val_accuracy: 0.8518 - val_recall: 0.7607 - val_precision: 0.6943 - val_AUROC: 0.9127 - val_AUPRC: 0.7405 - val_f1_score: 0.7260 - val_balanced_accuracy: 0.8221 - val_specificity: 0.8835 - val_miss_rate: 0.2393 - val_fall_out: 0.1165 - val_mcc: 0.6259
Epoch 16/1000
313/313 [==============================] - 12s 38ms/step - loss: 0.2912 - accuracy: 0.8650 - recall: 0.7856 - precision: 0.7180 - AUROC: 0.9280 - AUPRC: 0.7806 - f1_score: 0.7502 - balanced_accuracy: 0.8391 - specificity: 0.8926 - miss_rate: 0.2144 - fall_out: 0.1074 - mcc: 0.6593 - val_loss: 0.3198 - val_accuracy: 0.8512 - val_recall: 0.7737 - val_precision: 0.6884 - val_AUROC: 0.9127 - val_AUPRC: 0.7411 - val_f1_score: 0.7286 - val_balanced_accuracy: 0.8259 - val_specificity: 0.8781 - val_miss_rate: 0.2263 - val_fall_out: 0.1219 - val_mcc: 0.6286
Epoch 17/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.2885 - accuracy: 0.8665 - recall: 0.7824 - precision: 0.7230 - AUROC: 0.9295 - AUPRC: 0.7864 - f1_score: 0.7515 - balanced_accuracy: 0.8390 - specificity: 0.8957 - miss_rate: 0.2176 - fall_out: 0.1043 - mcc: 0.6614 - val_loss: 0.3230 - val_accuracy: 0.8516 - val_recall: 0.7126 - val_precision: 0.7126 - val_AUROC: 0.9121 - val_AUPRC: 0.7452 - val_f1_score: 0.7126 - val_balanced_accuracy: 0.8063 - val_specificity: 0.9000 - val_miss_rate: 0.2874 - val_fall_out: 0.1000 - val_mcc: 0.6126
Epoch 18/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2850 - accuracy: 0.8683 - recall: 0.7849 - precision: 0.7267 - AUROC: 0.9315 - AUPRC: 0.7914 - f1_score: 0.7547 - balanced_accuracy: 0.8411 - specificity: 0.8973 - miss_rate: 0.2151 - fall_out: 0.1027 - mcc: 0.6658 - val_loss: 0.3207 - val_accuracy: 0.8512 - val_recall: 0.7537 - val_precision: 0.6953 - val_AUROC: 0.9124 - val_AUPRC: 0.7419 - val_f1_score: 0.7234 - val_balanced_accuracy: 0.8194 - val_specificity: 0.8851 - val_miss_rate: 0.2463 - val_fall_out: 0.1149 - val_mcc: 0.6227
313/313 [==============================] - 7s 22ms/step - loss: 0.2632 - accuracy: 0.8808 - recall: 0.8121 - precision: 0.7479 - AUROC: 0.9441 - AUPRC: 0.8310 - f1_score: 0.7786 - balanced_accuracy: 0.8584 - specificity: 0.9047 - miss_rate: 0.1879 - fall_out: 0.0953 - mcc: 0.6984
79/79 [==============================] - 3s 40ms/step - loss: 0.3207 - accuracy: 0.8512 - recall: 0.7537 - precision: 0.6953 - AUROC: 0.9124 - AUPRC: 0.7419 - f1_score: 0.7234 - balanced_accuracy: 0.8194 - specificity: 0.8851 - miss_rate: 0.2463 - fall_out: 0.1149 - mcc: 0.6227
-- HOLDOUT 2 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 18s 51ms/step - loss: 12.0903 - accuracy: 0.8035 - recall: 0.5787 - precision: 0.6298 - AUROC: 0.8616 - AUPRC: 0.6257 - f1_score: 0.6032 - balanced_accuracy: 0.7302 - specificity: 0.8817 - miss_rate: 0.4213 - fall_out: 0.1183 - mcc: 0.4736 - val_loss: 2.3557 - val_accuracy: 0.8378 - val_recall: 0.7786 - val_precision: 0.6567 - val_AUROC: 0.8978 - val_AUPRC: 0.6934 - val_f1_score: 0.7124 - val_balanced_accuracy: 0.8185 - val_specificity: 0.8584 - val_miss_rate: 0.2214 - val_fall_out: 0.1416 - val_mcc: 0.6048
Epoch 2/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.9666 - accuracy: 0.8304 - recall: 0.7326 - precision: 0.6527 - AUROC: 0.8898 - AUPRC: 0.6743 - f1_score: 0.6904 - balanced_accuracy: 0.7985 - specificity: 0.8644 - miss_rate: 0.2674 - fall_out: 0.1356 - mcc: 0.5759 - val_loss: 1.5980 - val_accuracy: 0.8379 - val_recall: 0.8082 - val_precision: 0.6495 - val_AUROC: 0.9006 - val_AUPRC: 0.7010 - val_f1_score: 0.7202 - val_balanced_accuracy: 0.8282 - val_specificity: 0.8482 - val_miss_rate: 0.1918 - val_fall_out: 0.1518 - val_mcc: 0.6152
Epoch 3/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.7663 - accuracy: 0.8349 - recall: 0.7590 - precision: 0.6557 - AUROC: 0.8954 - AUPRC: 0.6866 - f1_score: 0.7036 - balanced_accuracy: 0.8102 - specificity: 0.8614 - miss_rate: 0.2410 - fall_out: 0.1386 - mcc: 0.5931 - val_loss: 1.9295 - val_accuracy: 0.8444 - val_recall: 0.7617 - val_precision: 0.6764 - val_AUROC: 0.9053 - val_AUPRC: 0.7153 - val_f1_score: 0.7165 - val_balanced_accuracy: 0.8174 - val_specificity: 0.8732 - val_miss_rate: 0.2383 - val_fall_out: 0.1268 - val_mcc: 0.6119
Epoch 4/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.6362 - accuracy: 0.8380 - recall: 0.7693 - precision: 0.6597 - AUROC: 0.8986 - AUPRC: 0.6948 - f1_score: 0.7103 - balanced_accuracy: 0.8156 - specificity: 0.8620 - miss_rate: 0.2307 - fall_out: 0.1380 - mcc: 0.6022 - val_loss: 1.3163 - val_accuracy: 0.8442 - val_recall: 0.7948 - val_precision: 0.6661 - val_AUROC: 0.9064 - val_AUPRC: 0.7166 - val_f1_score: 0.7248 - val_balanced_accuracy: 0.8281 - val_specificity: 0.8613 - val_miss_rate: 0.2052 - val_fall_out: 0.1387 - val_mcc: 0.6220
Epoch 5/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.5402 - accuracy: 0.8411 - recall: 0.7703 - precision: 0.6662 - AUROC: 0.9015 - AUPRC: 0.7023 - f1_score: 0.7145 - balanced_accuracy: 0.8180 - specificity: 0.8657 - miss_rate: 0.2297 - fall_out: 0.1343 - mcc: 0.6083 - val_loss: 1.7360 - val_accuracy: 0.8462 - val_recall: 0.7818 - val_precision: 0.6744 - val_AUROC: 0.9077 - val_AUPRC: 0.7230 - val_f1_score: 0.7241 - val_balanced_accuracy: 0.8252 - val_specificity: 0.8686 - val_miss_rate: 0.2182 - val_fall_out: 0.1314 - val_mcc: 0.6216
Epoch 6/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.4679 - accuracy: 0.8428 - recall: 0.7673 - precision: 0.6710 - AUROC: 0.9038 - AUPRC: 0.7080 - f1_score: 0.7159 - balanced_accuracy: 0.8182 - specificity: 0.8691 - miss_rate: 0.2327 - fall_out: 0.1309 - mcc: 0.6106 - val_loss: 1.2048 - val_accuracy: 0.8477 - val_recall: 0.7093 - val_precision: 0.7033 - val_AUROC: 0.9092 - val_AUPRC: 0.7263 - val_f1_score: 0.7063 - val_balanced_accuracy: 0.8026 - val_specificity: 0.8959 - val_miss_rate: 0.2907 - val_fall_out: 0.1041 - val_mcc: 0.6035
Epoch 7/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.4069 - accuracy: 0.8453 - recall: 0.7689 - precision: 0.6761 - AUROC: 0.9058 - AUPRC: 0.7111 - f1_score: 0.7196 - balanced_accuracy: 0.8204 - specificity: 0.8719 - miss_rate: 0.2311 - fall_out: 0.1281 - mcc: 0.6158 - val_loss: 1.5825 - val_accuracy: 0.8493 - val_recall: 0.7438 - val_precision: 0.6942 - val_AUROC: 0.9096 - val_AUPRC: 0.7284 - val_f1_score: 0.7182 - val_balanced_accuracy: 0.8149 - val_specificity: 0.8860 - val_miss_rate: 0.2562 - val_fall_out: 0.1140 - val_mcc: 0.6161
Epoch 8/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.3587 - accuracy: 0.8446 - recall: 0.7668 - precision: 0.6751 - AUROC: 0.9070 - AUPRC: 0.7157 - f1_score: 0.7181 - balanced_accuracy: 0.8192 - specificity: 0.8716 - miss_rate: 0.2332 - fall_out: 0.1284 - mcc: 0.6137 - val_loss: 1.1676 - val_accuracy: 0.8458 - val_recall: 0.7881 - val_precision: 0.6715 - val_AUROC: 0.9095 - val_AUPRC: 0.7270 - val_f1_score: 0.7251 - val_balanced_accuracy: 0.8270 - val_specificity: 0.8659 - val_miss_rate: 0.2119 - val_fall_out: 0.1341 - val_mcc: 0.6227
Epoch 9/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.3180 - accuracy: 0.8470 - recall: 0.7727 - precision: 0.6790 - AUROC: 0.9081 - AUPRC: 0.7165 - f1_score: 0.7228 - balanced_accuracy: 0.8228 - specificity: 0.8729 - miss_rate: 0.2273 - fall_out: 0.1271 - mcc: 0.6202 - val_loss: 1.4749 - val_accuracy: 0.8493 - val_recall: 0.6979 - val_precision: 0.7124 - val_AUROC: 0.9113 - val_AUPRC: 0.7324 - val_f1_score: 0.7051 - val_balanced_accuracy: 0.7999 - val_specificity: 0.9020 - val_miss_rate: 0.3021 - val_fall_out: 0.0980 - val_mcc: 0.6039
Epoch 10/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.2793 - accuracy: 0.8492 - recall: 0.7667 - precision: 0.6861 - AUROC: 0.9104 - AUPRC: 0.7286 - f1_score: 0.7242 - balanced_accuracy: 0.8223 - specificity: 0.8779 - miss_rate: 0.2333 - fall_out: 0.1221 - mcc: 0.6227 - val_loss: 1.1447 - val_accuracy: 0.8491 - val_recall: 0.7644 - val_precision: 0.6865 - val_AUROC: 0.9109 - val_AUPRC: 0.7306 - val_f1_score: 0.7234 - val_balanced_accuracy: 0.8215 - val_specificity: 0.8785 - val_miss_rate: 0.2356 - val_fall_out: 0.1215 - val_mcc: 0.6217
Epoch 11/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.2475 - accuracy: 0.8494 - recall: 0.7686 - precision: 0.6859 - AUROC: 0.9110 - AUPRC: 0.7265 - f1_score: 0.7249 - balanced_accuracy: 0.8231 - specificity: 0.8775 - miss_rate: 0.2314 - fall_out: 0.1225 - mcc: 0.6236 - val_loss: 1.3696 - val_accuracy: 0.8505 - val_recall: 0.7365 - val_precision: 0.7001 - val_AUROC: 0.9125 - val_AUPRC: 0.7337 - val_f1_score: 0.7178 - val_balanced_accuracy: 0.8133 - val_specificity: 0.8902 - val_miss_rate: 0.2635 - val_fall_out: 0.1098 - val_mcc: 0.6166
Epoch 12/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.2149 - accuracy: 0.8504 - recall: 0.7751 - precision: 0.6862 - AUROC: 0.9122 - AUPRC: 0.7305 - f1_score: 0.7279 - balanced_accuracy: 0.8259 - specificity: 0.8767 - miss_rate: 0.2249 - fall_out: 0.1233 - mcc: 0.6275 - val_loss: 1.1725 - val_accuracy: 0.8514 - val_recall: 0.7373 - val_precision: 0.7020 - val_AUROC: 0.9125 - val_AUPRC: 0.7345 - val_f1_score: 0.7192 - val_balanced_accuracy: 0.8142 - val_specificity: 0.8911 - val_miss_rate: 0.2627 - val_fall_out: 0.1089 - val_mcc: 0.6186
313/313 [==============================] - 8s 24ms/step - loss: 1.1601 - accuracy: 0.8567 - recall: 0.7512 - precision: 0.7104 - AUROC: 0.9212 - AUPRC: 0.7549 - f1_score: 0.7302 - balanced_accuracy: 0.8223 - specificity: 0.8935 - miss_rate: 0.2488 - fall_out: 0.1065 - mcc: 0.6333
79/79 [==============================] - 4s 43ms/step - loss: 1.1725 - accuracy: 0.8514 - recall: 0.7373 - precision: 0.7020 - AUROC: 0.9125 - AUPRC: 0.7345 - f1_score: 0.7192 - balanced_accuracy: 0.8142 - specificity: 0.8911 - miss_rate: 0.2627 - fall_out: 0.1089 - mcc: 0.6186
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_1\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_1\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 136, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.3, 'units_4': 40, 'dropout_4': 0.5}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_1\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_1\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 8, 'conv_dropout_0': 0.0, 'dense_depth': 2, 'units_0': 128, 'dense_dropout_0': 0.3, 'filters_1': 160, 'kernel_size_1': 16, 'conv_dropout_1': 0.0, 'filters_2': 8, 'kernel_size_2': 8, 'conv_dropout_2': 0.0, 'filters_3': 160, 'kernel_size_3': 16, 'conv_dropout_3': 0.2, 'filters_4': 144, 'kernel_size_4': 6, 'conv_dropout_4': 0.1, 'units_1': 64, 'dense_dropout_1': 0.3, 'units_2': 48, 'dense_dropout_2': 0.4, 'units_3': 16, 'dense_dropout_3': 0.4, 'filters_5': 144, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 1) 137
=================================================================
Total params: 76,569
Trainable params: 76,569
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 5280
dropout_4 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 160) 409760
dropout_5 (Dropout) (None, 128, 160) 0
max_pooling1d_2 (MaxPooling (None, 64, 160) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 10248
dropout_6 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
conv1d_4 (Conv1D) (None, 32, 160) 20640
dropout_7 (Dropout) (None, 32, 160) 0
max_pooling1d_4 (MaxPooling (None, 16, 160) 0
1D)
conv1d_5 (Conv1D) (None, 16, 144) 138384
dropout_8 (Dropout) (None, 16, 144) 0
max_pooling1d_5 (MaxPooling (None, 8, 144) 0
1D)
conv1d_6 (Conv1D) (None, 8, 144) 124560
dropout_9 (Dropout) (None, 8, 144) 0
max_pooling1d_6 (MaxPooling (None, 4, 144) 0
1D)
flatten_1 (Flatten) (None, 576) 0
dense_6 (Dense) (None, 128) 73856
dropout_10 (Dropout) (None, 128) 0
dense_7 (Dense) (None, 64) 8256
dropout_11 (Dropout) (None, 64) 0
dense_8 (Dense) (None, 1) 65
=================================================================
Total params: 791,049
Trainable params: 791,049
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 5280 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 160) 409760 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 160) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 160) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 10248 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 160) 20640 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 160) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 160) 0 ['dropout_7[0][0]']
conv1d_5 (Conv1D) (None, 16, 144) 138384 ['max_pooling1d_4[0][0]']
dropout_8 (Dropout) (None, 16, 144) 0 ['conv1d_5[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 144) 0 ['dropout_8[0][0]']
conv1d_6 (Conv1D) (None, 8, 144) 124560 ['max_pooling1d_5[0][0]']
dropout_9 (Dropout) (None, 8, 144) 0 ['conv1d_6[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 144) 0 ['dropout_9[0][0]']
flatten_1 (Flatten) (None, 576) 0 ['max_pooling1d_6[0][0]']
dense_6 (Dense) (None, 128) 73856 ['flatten_1[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
dropout_10 (Dropout) (None, 128) 0 ['dense_6[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
dense_7 (Dense) (None, 64) 8256 ['dropout_10[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_11 (Dropout) (None, 64) 0 ['dense_7[0][0]']
concatenate (Concatenate) (None, 200) 0 ['dropout_3[0][0]',
'dropout_11[0][0]']
dense_9 (Dense) (None, 64) 12864 ['concatenate[0][0]']
dense_10 (Dense) (None, 1) 65 ['dense_9[0][0]']
==================================================================================================
Total params: 880,345
Trainable params: 880,345
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 17s 48ms/step - loss: 0.3837 - accuracy: 0.8155 - recall: 0.6428 - precision: 0.6427 - AUROC: 0.8742 - AUPRC: 0.6433 - f1_score: 0.6427 - balanced_accuracy: 0.7592 - specificity: 0.8757 - miss_rate: 0.3572 - fall_out: 0.1243 - mcc: 0.5184 - val_loss: 0.3464 - val_accuracy: 0.8374 - val_recall: 0.7347 - val_precision: 0.6683 - val_AUROC: 0.8977 - val_AUPRC: 0.6961 - val_f1_score: 0.6999 - val_balanced_accuracy: 0.8039 - val_specificity: 0.8731 - val_miss_rate: 0.2653 - val_fall_out: 0.1269 - val_mcc: 0.5900
Epoch 2/1000
313/313 [==============================] - 15s 46ms/step - loss: 0.3468 - accuracy: 0.8356 - recall: 0.7262 - precision: 0.6668 - AUROC: 0.8953 - AUPRC: 0.6886 - f1_score: 0.6952 - balanced_accuracy: 0.7999 - specificity: 0.8737 - miss_rate: 0.2738 - fall_out: 0.1263 - mcc: 0.5840 - val_loss: 0.3369 - val_accuracy: 0.8414 - val_recall: 0.7805 - val_precision: 0.6640 - val_AUROC: 0.9036 - val_AUPRC: 0.7103 - val_f1_score: 0.7175 - val_balanced_accuracy: 0.8215 - val_specificity: 0.8626 - val_miss_rate: 0.2195 - val_fall_out: 0.1374 - val_mcc: 0.6121
Epoch 3/1000
313/313 [==============================] - 15s 46ms/step - loss: 0.3387 - accuracy: 0.8400 - recall: 0.7424 - precision: 0.6721 - AUROC: 0.9002 - AUPRC: 0.6986 - f1_score: 0.7055 - balanced_accuracy: 0.8082 - specificity: 0.8740 - miss_rate: 0.2576 - fall_out: 0.1260 - mcc: 0.5974 - val_loss: 0.3336 - val_accuracy: 0.8441 - val_recall: 0.7939 - val_precision: 0.6662 - val_AUROC: 0.9064 - val_AUPRC: 0.7165 - val_f1_score: 0.7245 - val_balanced_accuracy: 0.8277 - val_specificity: 0.8616 - val_miss_rate: 0.2061 - val_fall_out: 0.1384 - val_mcc: 0.6216
Epoch 4/1000
313/313 [==============================] - 15s 46ms/step - loss: 0.3322 - accuracy: 0.8428 - recall: 0.7399 - precision: 0.6794 - AUROC: 0.9046 - AUPRC: 0.7125 - f1_score: 0.7084 - balanced_accuracy: 0.8092 - specificity: 0.8785 - miss_rate: 0.2601 - fall_out: 0.1215 - mcc: 0.6020 - val_loss: 0.3278 - val_accuracy: 0.8473 - val_recall: 0.7755 - val_precision: 0.6787 - val_AUROC: 0.9087 - val_AUPRC: 0.7253 - val_f1_score: 0.7239 - val_balanced_accuracy: 0.8239 - val_specificity: 0.8723 - val_miss_rate: 0.2245 - val_fall_out: 0.1277 - val_mcc: 0.6216
Epoch 5/1000
313/313 [==============================] - 15s 46ms/step - loss: 0.3277 - accuracy: 0.8463 - recall: 0.7517 - precision: 0.6842 - AUROC: 0.9073 - AUPRC: 0.7207 - f1_score: 0.7163 - balanced_accuracy: 0.8155 - specificity: 0.8793 - miss_rate: 0.2483 - fall_out: 0.1207 - mcc: 0.6126 - val_loss: 0.3279 - val_accuracy: 0.8469 - val_recall: 0.7786 - val_precision: 0.6768 - val_AUROC: 0.9099 - val_AUPRC: 0.7243 - val_f1_score: 0.7241 - val_balanced_accuracy: 0.8246 - val_specificity: 0.8706 - val_miss_rate: 0.2214 - val_fall_out: 0.1294 - val_mcc: 0.6218
Epoch 6/1000
313/313 [==============================] - 15s 46ms/step - loss: 0.3227 - accuracy: 0.8485 - recall: 0.7535 - precision: 0.6887 - AUROC: 0.9101 - AUPRC: 0.7307 - f1_score: 0.7196 - balanced_accuracy: 0.8175 - specificity: 0.8815 - miss_rate: 0.2465 - fall_out: 0.1185 - mcc: 0.6172 - val_loss: 0.3228 - val_accuracy: 0.8470 - val_recall: 0.7919 - val_precision: 0.6731 - val_AUROC: 0.9114 - val_AUPRC: 0.7382 - val_f1_score: 0.7277 - val_balanced_accuracy: 0.8291 - val_specificity: 0.8662 - val_miss_rate: 0.2081 - val_fall_out: 0.1338 - val_mcc: 0.6263
Epoch 7/1000
313/313 [==============================] - 15s 46ms/step - loss: 0.3178 - accuracy: 0.8521 - recall: 0.7590 - precision: 0.6956 - AUROC: 0.9135 - AUPRC: 0.7427 - f1_score: 0.7259 - balanced_accuracy: 0.8217 - specificity: 0.8844 - miss_rate: 0.2410 - fall_out: 0.1156 - mcc: 0.6260 - val_loss: 0.3187 - val_accuracy: 0.8523 - val_recall: 0.7413 - val_precision: 0.7028 - val_AUROC: 0.9140 - val_AUPRC: 0.7473 - val_f1_score: 0.7215 - val_balanced_accuracy: 0.8161 - val_specificity: 0.8909 - val_miss_rate: 0.2587 - val_fall_out: 0.1091 - val_mcc: 0.6215
Epoch 8/1000
313/313 [==============================] - 15s 46ms/step - loss: 0.3099 - accuracy: 0.8562 - recall: 0.7593 - precision: 0.7059 - AUROC: 0.9184 - AUPRC: 0.7588 - f1_score: 0.7316 - balanced_accuracy: 0.8246 - specificity: 0.8899 - miss_rate: 0.2407 - fall_out: 0.1101 - mcc: 0.6344 - val_loss: 0.3205 - val_accuracy: 0.8520 - val_recall: 0.7646 - val_precision: 0.6936 - val_AUROC: 0.9125 - val_AUPRC: 0.7457 - val_f1_score: 0.7274 - val_balanced_accuracy: 0.8235 - val_specificity: 0.8825 - val_miss_rate: 0.2354 - val_fall_out: 0.1175 - val_mcc: 0.6275
Epoch 9/1000
313/313 [==============================] - 15s 46ms/step - loss: 0.2966 - accuracy: 0.8644 - recall: 0.7623 - precision: 0.7262 - AUROC: 0.9262 - AUPRC: 0.7865 - f1_score: 0.7438 - balanced_accuracy: 0.8311 - specificity: 0.9000 - miss_rate: 0.2377 - fall_out: 0.1000 - mcc: 0.6521 - val_loss: 0.3242 - val_accuracy: 0.8488 - val_recall: 0.6812 - val_precision: 0.7184 - val_AUROC: 0.9119 - val_AUPRC: 0.7427 - val_f1_score: 0.6993 - val_balanced_accuracy: 0.7941 - val_specificity: 0.9071 - val_miss_rate: 0.3188 - val_fall_out: 0.0929 - val_mcc: 0.5988
313/313 [==============================] - 7s 21ms/step - loss: 0.2676 - accuracy: 0.8856 - recall: 0.7584 - precision: 0.7900 - AUROC: 0.9453 - AUPRC: 0.8497 - f1_score: 0.7738 - balanced_accuracy: 0.8441 - specificity: 0.9299 - miss_rate: 0.2416 - fall_out: 0.0701 - mcc: 0.6976
79/79 [==============================] - 4s 42ms/step - loss: 0.3242 - accuracy: 0.8488 - recall: 0.6812 - precision: 0.7184 - AUROC: 0.9119 - AUPRC: 0.7427 - f1_score: 0.6993 - balanced_accuracy: 0.7941 - specificity: 0.9071 - miss_rate: 0.3188 - fall_out: 0.0929 - mcc: 0.5988
-- HOLDOUT 3 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 18s 51ms/step - loss: 11.8673 - accuracy: 0.8058 - recall: 0.5775 - precision: 0.6364 - AUROC: 0.8623 - AUPRC: 0.6295 - f1_score: 0.6055 - balanced_accuracy: 0.7313 - specificity: 0.8852 - miss_rate: 0.4225 - fall_out: 0.1148 - mcc: 0.4781 - val_loss: 2.2373 - val_accuracy: 0.8386 - val_recall: 0.7904 - val_precision: 0.6553 - val_AUROC: 0.8995 - val_AUPRC: 0.6950 - val_f1_score: 0.7165 - val_balanced_accuracy: 0.8229 - val_specificity: 0.8553 - val_miss_rate: 0.2096 - val_fall_out: 0.1447 - val_mcc: 0.6102
Epoch 2/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.9711 - accuracy: 0.8295 - recall: 0.7316 - precision: 0.6510 - AUROC: 0.8907 - AUPRC: 0.6762 - f1_score: 0.6889 - balanced_accuracy: 0.7976 - specificity: 0.8635 - miss_rate: 0.2684 - fall_out: 0.1365 - mcc: 0.5739 - val_loss: 1.6622 - val_accuracy: 0.8408 - val_recall: 0.8059 - val_precision: 0.6560 - val_AUROC: 0.9040 - val_AUPRC: 0.7070 - val_f1_score: 0.7233 - val_balanced_accuracy: 0.8294 - val_specificity: 0.8530 - val_miss_rate: 0.1941 - val_fall_out: 0.1470 - val_mcc: 0.6196
Epoch 3/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.7711 - accuracy: 0.8369 - recall: 0.7616 - precision: 0.6593 - AUROC: 0.8968 - AUPRC: 0.6890 - f1_score: 0.7067 - balanced_accuracy: 0.8123 - specificity: 0.8631 - miss_rate: 0.2384 - fall_out: 0.1369 - mcc: 0.5975 - val_loss: 1.9249 - val_accuracy: 0.8437 - val_recall: 0.7970 - val_precision: 0.6645 - val_AUROC: 0.9064 - val_AUPRC: 0.7129 - val_f1_score: 0.7247 - val_balanced_accuracy: 0.8285 - val_specificity: 0.8600 - val_miss_rate: 0.2030 - val_fall_out: 0.1400 - val_mcc: 0.6218
Epoch 4/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.6414 - accuracy: 0.8404 - recall: 0.7602 - precision: 0.6677 - AUROC: 0.9001 - AUPRC: 0.6994 - f1_score: 0.7110 - balanced_accuracy: 0.8143 - specificity: 0.8684 - miss_rate: 0.2398 - fall_out: 0.1316 - mcc: 0.6038 - val_loss: 1.4216 - val_accuracy: 0.8453 - val_recall: 0.7786 - val_precision: 0.6732 - val_AUROC: 0.9075 - val_AUPRC: 0.7142 - val_f1_score: 0.7221 - val_balanced_accuracy: 0.8235 - val_specificity: 0.8685 - val_miss_rate: 0.2214 - val_fall_out: 0.1315 - val_mcc: 0.6188
Epoch 5/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.5454 - accuracy: 0.8421 - recall: 0.7687 - precision: 0.6689 - AUROC: 0.9023 - AUPRC: 0.7045 - f1_score: 0.7153 - balanced_accuracy: 0.8181 - specificity: 0.8676 - miss_rate: 0.2313 - fall_out: 0.1324 - mcc: 0.6096 - val_loss: 1.6962 - val_accuracy: 0.8459 - val_recall: 0.7836 - val_precision: 0.6731 - val_AUROC: 0.9086 - val_AUPRC: 0.7160 - val_f1_score: 0.7241 - val_balanced_accuracy: 0.8256 - val_specificity: 0.8675 - val_miss_rate: 0.2164 - val_fall_out: 0.1325 - val_mcc: 0.6215
Epoch 6/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.4716 - accuracy: 0.8431 - recall: 0.7633 - precision: 0.6728 - AUROC: 0.9037 - AUPRC: 0.7083 - f1_score: 0.7152 - balanced_accuracy: 0.8171 - specificity: 0.8709 - miss_rate: 0.2367 - fall_out: 0.1291 - mcc: 0.6099 - val_loss: 1.2368 - val_accuracy: 0.8462 - val_recall: 0.7863 - val_precision: 0.6730 - val_AUROC: 0.9084 - val_AUPRC: 0.7165 - val_f1_score: 0.7253 - val_balanced_accuracy: 0.8267 - val_specificity: 0.8671 - val_miss_rate: 0.2137 - val_fall_out: 0.1329 - val_mcc: 0.6230
Epoch 7/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.4098 - accuracy: 0.8455 - recall: 0.7664 - precision: 0.6773 - AUROC: 0.9062 - AUPRC: 0.7145 - f1_score: 0.7191 - balanced_accuracy: 0.8197 - specificity: 0.8730 - miss_rate: 0.2336 - fall_out: 0.1270 - mcc: 0.6153 - val_loss: 1.5136 - val_accuracy: 0.8481 - val_recall: 0.7865 - val_precision: 0.6772 - val_AUROC: 0.9104 - val_AUPRC: 0.7218 - val_f1_score: 0.7278 - val_balanced_accuracy: 0.8280 - val_specificity: 0.8696 - val_miss_rate: 0.2135 - val_fall_out: 0.1304 - val_mcc: 0.6266
Epoch 8/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.3602 - accuracy: 0.8455 - recall: 0.7686 - precision: 0.6768 - AUROC: 0.9074 - AUPRC: 0.7173 - f1_score: 0.7198 - balanced_accuracy: 0.8204 - specificity: 0.8723 - miss_rate: 0.2314 - fall_out: 0.1277 - mcc: 0.6161 - val_loss: 1.1457 - val_accuracy: 0.8492 - val_recall: 0.7718 - val_precision: 0.6843 - val_AUROC: 0.9105 - val_AUPRC: 0.7243 - val_f1_score: 0.7254 - val_balanced_accuracy: 0.8239 - val_specificity: 0.8761 - val_miss_rate: 0.2282 - val_fall_out: 0.1239 - val_mcc: 0.6241
Epoch 9/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.3159 - accuracy: 0.8470 - recall: 0.7681 - precision: 0.6803 - AUROC: 0.9093 - AUPRC: 0.7224 - f1_score: 0.7215 - balanced_accuracy: 0.8212 - specificity: 0.8744 - miss_rate: 0.2319 - fall_out: 0.1256 - mcc: 0.6187 - val_loss: 1.4662 - val_accuracy: 0.8517 - val_recall: 0.7584 - val_precision: 0.6949 - val_AUROC: 0.9114 - val_AUPRC: 0.7279 - val_f1_score: 0.7253 - val_balanced_accuracy: 0.8213 - val_specificity: 0.8841 - val_miss_rate: 0.2416 - val_fall_out: 0.1159 - val_mcc: 0.6251
Epoch 10/1000
313/313 [==============================] - 16s 49ms/step - loss: 1.2813 - accuracy: 0.8483 - recall: 0.7651 - precision: 0.6844 - AUROC: 0.9105 - AUPRC: 0.7238 - f1_score: 0.7225 - balanced_accuracy: 0.8212 - specificity: 0.8772 - miss_rate: 0.2349 - fall_out: 0.1228 - mcc: 0.6204 - val_loss: 1.1891 - val_accuracy: 0.8506 - val_recall: 0.7363 - val_precision: 0.7003 - val_AUROC: 0.9115 - val_AUPRC: 0.7251 - val_f1_score: 0.7178 - val_balanced_accuracy: 0.8133 - val_specificity: 0.8904 - val_miss_rate: 0.2637 - val_fall_out: 0.1096 - val_mcc: 0.6167
313/313 [==============================] - 8s 25ms/step - loss: 1.1796 - accuracy: 0.8540 - recall: 0.7380 - precision: 0.7086 - AUROC: 0.9190 - AUPRC: 0.7482 - f1_score: 0.7230 - balanced_accuracy: 0.8162 - specificity: 0.8944 - miss_rate: 0.2620 - fall_out: 0.1056 - mcc: 0.6242
79/79 [==============================] - 4s 46ms/step - loss: 1.1891 - accuracy: 0.8506 - recall: 0.7363 - precision: 0.7003 - AUROC: 0.9115 - AUPRC: 0.7251 - f1_score: 0.7178 - balanced_accuracy: 0.8133 - specificity: 0.8904 - miss_rate: 0.2637 - fall_out: 0.1096 - mcc: 0.6167
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_2\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_2\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 8, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.5}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_2\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_2\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 3, 'filters_0': 80, 'kernel_size_0': 8, 'conv_dropout_0': 0.0, 'dense_depth': 1, 'units_0': 64, 'dense_dropout_0': 0.5, 'filters_1': 8, 'kernel_size_1': 16, 'conv_dropout_1': 0.1, 'units_1': 48, 'dense_dropout_1': 0.3, 'units_2': 16, 'dense_dropout_2': 0.4, 'units_3': 16, 'dense_dropout_3': 0.3, 'filters_2': 8, 'kernel_size_2': 6, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.0, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.0}
- Removing uncorrelated/correlated features: ['ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 1) 137
=================================================================
Total params: 76,569
Trainable params: 76,569
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 80) 2640
dropout_4 (Dropout) (None, 256, 80) 0
max_pooling1d_1 (MaxPooling (None, 128, 80) 0
1D)
conv1d_2 (Conv1D) (None, 128, 8) 10248
dropout_5 (Dropout) (None, 128, 8) 0
max_pooling1d_2 (MaxPooling (None, 64, 8) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 392
dropout_6 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
flatten_1 (Flatten) (None, 256) 0
dense_6 (Dense) (None, 64) 16448
dropout_7 (Dropout) (None, 64) 0
dense_7 (Dense) (None, 1) 65
=================================================================
Total params: 29,793
Trainable params: 29,793
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 80) 2640 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 80) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 80) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 8) 10248 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 8) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 8) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 392 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_6[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
flatten_1 (Flatten) (None, 256) 0 ['max_pooling1d_3[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
dense_6 (Dense) (None, 64) 16448 ['flatten_1[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_7 (Dropout) (None, 64) 0 ['dense_6[0][0]']
concatenate (Concatenate) (None, 200) 0 ['dropout_3[0][0]',
'dropout_7[0][0]']
dense_8 (Dense) (None, 64) 12864 ['concatenate[0][0]']
dense_9 (Dense) (None, 1) 65 ['dense_8[0][0]']
==================================================================================================
Total params: 119,089
Trainable params: 119,089
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 11s 31ms/step - loss: 0.3694 - accuracy: 0.8244 - recall: 0.6758 - precision: 0.6548 - AUROC: 0.8827 - AUPRC: 0.6624 - f1_score: 0.6651 - balanced_accuracy: 0.7759 - specificity: 0.8761 - miss_rate: 0.3242 - fall_out: 0.1239 - mcc: 0.5463 - val_loss: 0.3428 - val_accuracy: 0.8350 - val_recall: 0.7246 - val_precision: 0.6657 - val_AUROC: 0.8998 - val_AUPRC: 0.6956 - val_f1_score: 0.6939 - val_balanced_accuracy: 0.7990 - val_specificity: 0.8733 - val_miss_rate: 0.2754 - val_fall_out: 0.1267 - val_mcc: 0.5822
Epoch 2/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.3375 - accuracy: 0.8397 - recall: 0.7403 - precision: 0.6721 - AUROC: 0.9013 - AUPRC: 0.7048 - f1_score: 0.7045 - balanced_accuracy: 0.8073 - specificity: 0.8743 - miss_rate: 0.2597 - fall_out: 0.1257 - mcc: 0.5962 - val_loss: 0.3301 - val_accuracy: 0.8462 - val_recall: 0.7597 - val_precision: 0.6812 - val_AUROC: 0.9063 - val_AUPRC: 0.7131 - val_f1_score: 0.7183 - val_balanced_accuracy: 0.8180 - val_specificity: 0.8762 - val_miss_rate: 0.2403 - val_fall_out: 0.1238 - val_mcc: 0.6147
Epoch 3/1000
313/313 [==============================] - 11s 32ms/step - loss: 0.3295 - accuracy: 0.8449 - recall: 0.7422 - precision: 0.6838 - AUROC: 0.9064 - AUPRC: 0.7184 - f1_score: 0.7118 - balanced_accuracy: 0.8114 - specificity: 0.8806 - miss_rate: 0.2578 - fall_out: 0.1194 - mcc: 0.6068 - val_loss: 0.3269 - val_accuracy: 0.8493 - val_recall: 0.7450 - val_precision: 0.6939 - val_AUROC: 0.9091 - val_AUPRC: 0.7209 - val_f1_score: 0.7185 - val_balanced_accuracy: 0.8153 - val_specificity: 0.8856 - val_miss_rate: 0.2550 - val_fall_out: 0.1144 - val_mcc: 0.6166
Epoch 4/1000
313/313 [==============================] - 10s 32ms/step - loss: 0.3228 - accuracy: 0.8482 - recall: 0.7530 - precision: 0.6883 - AUROC: 0.9106 - AUPRC: 0.7279 - f1_score: 0.7192 - balanced_accuracy: 0.8172 - specificity: 0.8814 - miss_rate: 0.2470 - fall_out: 0.1186 - mcc: 0.6167 - val_loss: 0.3217 - val_accuracy: 0.8505 - val_recall: 0.7640 - val_precision: 0.6901 - val_AUROC: 0.9119 - val_AUPRC: 0.7299 - val_f1_score: 0.7252 - val_balanced_accuracy: 0.8223 - val_specificity: 0.8806 - val_miss_rate: 0.2360 - val_fall_out: 0.1194 - val_mcc: 0.6244
Epoch 5/1000
313/313 [==============================] - 10s 32ms/step - loss: 0.3172 - accuracy: 0.8510 - recall: 0.7539 - precision: 0.6947 - AUROC: 0.9135 - AUPRC: 0.7361 - f1_score: 0.7231 - balanced_accuracy: 0.8193 - specificity: 0.8847 - miss_rate: 0.2461 - fall_out: 0.1153 - mcc: 0.6224 - val_loss: 0.3170 - val_accuracy: 0.8493 - val_recall: 0.7733 - val_precision: 0.6840 - val_AUROC: 0.9134 - val_AUPRC: 0.7337 - val_f1_score: 0.7259 - val_balanced_accuracy: 0.8245 - val_specificity: 0.8757 - val_miss_rate: 0.2267 - val_fall_out: 0.1243 - val_mcc: 0.6248
Epoch 6/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.3124 - accuracy: 0.8537 - recall: 0.7594 - precision: 0.6995 - AUROC: 0.9165 - AUPRC: 0.7474 - f1_score: 0.7282 - balanced_accuracy: 0.8229 - specificity: 0.8865 - miss_rate: 0.2406 - fall_out: 0.1135 - mcc: 0.6293 - val_loss: 0.3152 - val_accuracy: 0.8540 - val_recall: 0.7483 - val_precision: 0.7045 - val_AUROC: 0.9147 - val_AUPRC: 0.7376 - val_f1_score: 0.7257 - val_balanced_accuracy: 0.8195 - val_specificity: 0.8908 - val_miss_rate: 0.2517 - val_fall_out: 0.1092 - val_mcc: 0.6269
Epoch 7/1000
313/313 [==============================] - 10s 32ms/step - loss: 0.3058 - accuracy: 0.8574 - recall: 0.7596 - precision: 0.7088 - AUROC: 0.9205 - AUPRC: 0.7592 - f1_score: 0.7333 - balanced_accuracy: 0.8255 - specificity: 0.8914 - miss_rate: 0.2404 - fall_out: 0.1086 - mcc: 0.6369 - val_loss: 0.3117 - val_accuracy: 0.8546 - val_recall: 0.7667 - val_precision: 0.6991 - val_AUROC: 0.9173 - val_AUPRC: 0.7488 - val_f1_score: 0.7313 - val_balanced_accuracy: 0.8259 - val_specificity: 0.8852 - val_miss_rate: 0.2333 - val_fall_out: 0.1148 - val_mcc: 0.6332
Epoch 8/1000
313/313 [==============================] - 10s 32ms/step - loss: 0.2998 - accuracy: 0.8610 - recall: 0.7630 - precision: 0.7167 - AUROC: 0.9239 - AUPRC: 0.7707 - f1_score: 0.7391 - balanced_accuracy: 0.8290 - specificity: 0.8951 - miss_rate: 0.2370 - fall_out: 0.1049 - mcc: 0.6450 - val_loss: 0.3125 - val_accuracy: 0.8536 - val_recall: 0.7813 - val_precision: 0.6917 - val_AUROC: 0.9171 - val_AUPRC: 0.7530 - val_f1_score: 0.7337 - val_balanced_accuracy: 0.8300 - val_specificity: 0.8788 - val_miss_rate: 0.2187 - val_fall_out: 0.1212 - val_mcc: 0.6356
Epoch 9/1000
313/313 [==============================] - 10s 32ms/step - loss: 0.2934 - accuracy: 0.8640 - recall: 0.7690 - precision: 0.7223 - AUROC: 0.9275 - AUPRC: 0.7817 - f1_score: 0.7449 - balanced_accuracy: 0.8331 - specificity: 0.8971 - miss_rate: 0.2310 - fall_out: 0.1029 - mcc: 0.6529 - val_loss: 0.3144 - val_accuracy: 0.8533 - val_recall: 0.8206 - val_precision: 0.6785 - val_AUROC: 0.9180 - val_AUPRC: 0.7553 - val_f1_score: 0.7428 - val_balanced_accuracy: 0.8427 - val_specificity: 0.8647 - val_miss_rate: 0.1794 - val_fall_out: 0.1353 - val_mcc: 0.6472
313/313 [==============================] - 6s 19ms/step - loss: 0.2788 - accuracy: 0.8717 - recall: 0.8497 - precision: 0.7103 - AUROC: 0.9387 - AUPRC: 0.8238 - f1_score: 0.7738 - balanced_accuracy: 0.8646 - specificity: 0.8794 - miss_rate: 0.1503 - fall_out: 0.1206 - mcc: 0.6906
79/79 [==============================] - 4s 44ms/step - loss: 0.3144 - accuracy: 0.8533 - recall: 0.8206 - precision: 0.6785 - AUROC: 0.9180 - AUPRC: 0.7553 - f1_score: 0.7428 - balanced_accuracy: 0.8427 - specificity: 0.8647 - miss_rate: 0.1794 - fall_out: 0.1353 - mcc: 0.6472
-- HOLDOUT 4 --
-- 3 Uncorrelated features: [Pearson+Spearman]
['NBN', 'ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['NBN', 'ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 18s 51ms/step - loss: 11.8730 - accuracy: 0.7974 - recall: 0.5400 - precision: 0.6242 - AUROC: 0.8558 - AUPRC: 0.6155 - f1_score: 0.5791 - balanced_accuracy: 0.7135 - specificity: 0.8869 - miss_rate: 0.4600 - fall_out: 0.1131 - mcc: 0.4486 - val_loss: 1.9034 - val_accuracy: 0.8324 - val_recall: 0.6027 - val_precision: 0.7052 - val_AUROC: 0.9031 - val_AUPRC: 0.7026 - val_f1_score: 0.6499 - val_balanced_accuracy: 0.7575 - val_specificity: 0.9123 - val_miss_rate: 0.3973 - val_fall_out: 0.0877 - val_mcc: 0.5435
Epoch 2/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.9389 - accuracy: 0.8292 - recall: 0.6904 - precision: 0.6621 - AUROC: 0.8897 - AUPRC: 0.6745 - f1_score: 0.6760 - balanced_accuracy: 0.7839 - specificity: 0.8774 - miss_rate: 0.3096 - fall_out: 0.1226 - mcc: 0.5603 - val_loss: 1.9856 - val_accuracy: 0.8430 - val_recall: 0.7235 - val_precision: 0.6857 - val_AUROC: 0.9020 - val_AUPRC: 0.6841 - val_f1_score: 0.7041 - val_balanced_accuracy: 0.8040 - val_specificity: 0.8846 - val_miss_rate: 0.2765 - val_fall_out: 0.1154 - val_mcc: 0.5978
Epoch 3/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.7498 - accuracy: 0.8351 - recall: 0.7294 - precision: 0.6646 - AUROC: 0.8952 - AUPRC: 0.6865 - f1_score: 0.6955 - balanced_accuracy: 0.8007 - specificity: 0.8719 - miss_rate: 0.2706 - fall_out: 0.1281 - mcc: 0.5840 - val_loss: 1.4593 - val_accuracy: 0.8436 - val_recall: 0.7332 - val_precision: 0.6837 - val_AUROC: 0.9076 - val_AUPRC: 0.7100 - val_f1_score: 0.7076 - val_balanced_accuracy: 0.8076 - val_specificity: 0.8820 - val_miss_rate: 0.2668 - val_fall_out: 0.1180 - val_mcc: 0.6017
Epoch 4/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.6225 - accuracy: 0.8378 - recall: 0.7455 - precision: 0.6660 - AUROC: 0.8985 - AUPRC: 0.6946 - f1_score: 0.7035 - balanced_accuracy: 0.8077 - specificity: 0.8699 - miss_rate: 0.2545 - fall_out: 0.1301 - mcc: 0.5941 - val_loss: 1.7643 - val_accuracy: 0.8468 - val_recall: 0.7537 - val_precision: 0.6846 - val_AUROC: 0.9090 - val_AUPRC: 0.7130 - val_f1_score: 0.7175 - val_balanced_accuracy: 0.8164 - val_specificity: 0.8791 - val_miss_rate: 0.2463 - val_fall_out: 0.1209 - val_mcc: 0.6140
Epoch 5/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.5317 - accuracy: 0.8415 - recall: 0.7518 - precision: 0.6726 - AUROC: 0.9009 - AUPRC: 0.6996 - f1_score: 0.7100 - balanced_accuracy: 0.8122 - specificity: 0.8727 - miss_rate: 0.2482 - fall_out: 0.1273 - mcc: 0.6031 - val_loss: 1.3043 - val_accuracy: 0.8471 - val_recall: 0.7440 - val_precision: 0.6886 - val_AUROC: 0.9094 - val_AUPRC: 0.7162 - val_f1_score: 0.7153 - val_balanced_accuracy: 0.8135 - val_specificity: 0.8829 - val_miss_rate: 0.2560 - val_fall_out: 0.1171 - val_mcc: 0.6118
Epoch 6/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.4612 - accuracy: 0.8421 - recall: 0.7501 - precision: 0.6746 - AUROC: 0.9022 - AUPRC: 0.7028 - f1_score: 0.7103 - balanced_accuracy: 0.8121 - specificity: 0.8741 - miss_rate: 0.2499 - fall_out: 0.1259 - mcc: 0.6038 - val_loss: 1.6431 - val_accuracy: 0.8491 - val_recall: 0.7615 - val_precision: 0.6875 - val_AUROC: 0.9093 - val_AUPRC: 0.7143 - val_f1_score: 0.7226 - val_balanced_accuracy: 0.8205 - val_specificity: 0.8796 - val_miss_rate: 0.2385 - val_fall_out: 0.1204 - val_mcc: 0.6208
Epoch 7/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.4058 - accuracy: 0.8443 - recall: 0.7564 - precision: 0.6778 - AUROC: 0.9046 - AUPRC: 0.7083 - f1_score: 0.7149 - balanced_accuracy: 0.8156 - specificity: 0.8749 - miss_rate: 0.2436 - fall_out: 0.1251 - mcc: 0.6100 - val_loss: 1.1595 - val_accuracy: 0.8488 - val_recall: 0.7345 - val_precision: 0.6965 - val_AUROC: 0.9123 - val_AUPRC: 0.7264 - val_f1_score: 0.7150 - val_balanced_accuracy: 0.8116 - val_specificity: 0.8886 - val_miss_rate: 0.2655 - val_fall_out: 0.1114 - val_mcc: 0.6126
Epoch 8/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.3581 - accuracy: 0.8461 - recall: 0.7577 - precision: 0.6815 - AUROC: 0.9064 - AUPRC: 0.7147 - f1_score: 0.7176 - balanced_accuracy: 0.8173 - specificity: 0.8768 - miss_rate: 0.2423 - fall_out: 0.1232 - mcc: 0.6138 - val_loss: 1.5187 - val_accuracy: 0.8515 - val_recall: 0.7532 - val_precision: 0.6963 - val_AUROC: 0.9116 - val_AUPRC: 0.7265 - val_f1_score: 0.7236 - val_balanced_accuracy: 0.8194 - val_specificity: 0.8857 - val_miss_rate: 0.2468 - val_fall_out: 0.1143 - val_mcc: 0.6232
Epoch 9/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.3203 - accuracy: 0.8462 - recall: 0.7497 - precision: 0.6846 - AUROC: 0.9074 - AUPRC: 0.7180 - f1_score: 0.7157 - balanced_accuracy: 0.8147 - specificity: 0.8798 - miss_rate: 0.2503 - fall_out: 0.1202 - mcc: 0.6118 - val_loss: 1.1532 - val_accuracy: 0.8509 - val_recall: 0.7861 - val_precision: 0.6836 - val_AUROC: 0.9125 - val_AUPRC: 0.7276 - val_f1_score: 0.7313 - val_balanced_accuracy: 0.8298 - val_specificity: 0.8734 - val_miss_rate: 0.2139 - val_fall_out: 0.1266 - val_mcc: 0.6317
Epoch 10/1000
313/313 [==============================] - 16s 51ms/step - loss: 1.2804 - accuracy: 0.8483 - recall: 0.7570 - precision: 0.6872 - AUROC: 0.9095 - AUPRC: 0.7211 - f1_score: 0.7204 - balanced_accuracy: 0.8186 - specificity: 0.8801 - miss_rate: 0.2430 - fall_out: 0.1199 - mcc: 0.6181 - val_loss: 1.4031 - val_accuracy: 0.8493 - val_recall: 0.6915 - val_precision: 0.7153 - val_AUROC: 0.9138 - val_AUPRC: 0.7321 - val_f1_score: 0.7032 - val_balanced_accuracy: 0.7979 - val_specificity: 0.9043 - val_miss_rate: 0.3085 - val_fall_out: 0.0957 - val_mcc: 0.6024
Epoch 11/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.2495 - accuracy: 0.8489 - recall: 0.7566 - precision: 0.6886 - AUROC: 0.9095 - AUPRC: 0.7218 - f1_score: 0.7210 - balanced_accuracy: 0.8188 - specificity: 0.8810 - miss_rate: 0.2434 - fall_out: 0.1190 - mcc: 0.6190 - val_loss: 1.0528 - val_accuracy: 0.8492 - val_recall: 0.7518 - val_precision: 0.6911 - val_AUROC: 0.9136 - val_AUPRC: 0.7309 - val_f1_score: 0.7202 - val_balanced_accuracy: 0.8174 - val_specificity: 0.8831 - val_miss_rate: 0.2482 - val_fall_out: 0.1169 - val_mcc: 0.6182
Epoch 12/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.2162 - accuracy: 0.8506 - recall: 0.7573 - precision: 0.6927 - AUROC: 0.9114 - AUPRC: 0.7305 - f1_score: 0.7236 - balanced_accuracy: 0.8202 - specificity: 0.8831 - miss_rate: 0.2427 - fall_out: 0.1169 - mcc: 0.6227 - val_loss: 1.2755 - val_accuracy: 0.8491 - val_recall: 0.7208 - val_precision: 0.7025 - val_AUROC: 0.9140 - val_AUPRC: 0.7363 - val_f1_score: 0.7115 - val_balanced_accuracy: 0.8073 - val_specificity: 0.8938 - val_miss_rate: 0.2792 - val_fall_out: 0.1062 - val_mcc: 0.6095
Epoch 13/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.1874 - accuracy: 0.8506 - recall: 0.7688 - precision: 0.6885 - AUROC: 0.9122 - AUPRC: 0.7289 - f1_score: 0.7265 - balanced_accuracy: 0.8239 - specificity: 0.8790 - miss_rate: 0.2312 - fall_out: 0.1210 - mcc: 0.6259 - val_loss: 1.1669 - val_accuracy: 0.8519 - val_recall: 0.7553 - val_precision: 0.6967 - val_AUROC: 0.9151 - val_AUPRC: 0.7393 - val_f1_score: 0.7248 - val_balanced_accuracy: 0.8204 - val_specificity: 0.8856 - val_miss_rate: 0.2447 - val_fall_out: 0.1144 - val_mcc: 0.6247
313/313 [==============================] - 8s 24ms/step - loss: 1.1562 - accuracy: 0.8586 - recall: 0.7700 - precision: 0.7077 - AUROC: 0.9224 - AUPRC: 0.7561 - f1_score: 0.7376 - balanced_accuracy: 0.8297 - specificity: 0.8894 - miss_rate: 0.2300 - fall_out: 0.1106 - mcc: 0.6420
79/79 [==============================] - 4s 47ms/step - loss: 1.1669 - accuracy: 0.8519 - recall: 0.7553 - precision: 0.6967 - AUROC: 0.9151 - AUPRC: 0.7393 - f1_score: 0.7248 - balanced_accuracy: 0.8204 - specificity: 0.8856 - miss_rate: 0.2447 - fall_out: 0.1144 - mcc: 0.6247
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_3\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_3\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 40, 'dropout_0': 0.5, 'units_1': 72, 'dropout_1': 0.5, 'units_2': 136, 'dropout_2': 0.5}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_3\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_3\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 6, 'conv_dropout_0': 0.1, 'dense_depth': 1, 'units_0': 128, 'dense_dropout_0': 0.5, 'filters_1': 160, 'kernel_size_1': 6, 'conv_dropout_1': 0.0, 'filters_2': 64, 'kernel_size_2': 16, 'conv_dropout_2': 0.0, 'units_1': 48, 'dense_dropout_1': 0.5, 'units_2': 80, 'dense_dropout_2': 0.5, 'units_3': 48, 'dense_dropout_3': 0.5, 'filters_3': 8, 'kernel_size_3': 8, 'conv_dropout_3': 0.1, 'filters_4': 32, 'kernel_size_4': 12, 'conv_dropout_4': 0.2, 'filters_5': 112, 'kernel_size_5': 8, 'conv_dropout_5': 0.0, 'units_4': 32, 'dense_dropout_4': 0.4}
- Removing uncorrelated/correlated features: ['ZNF207', 'NBN', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 40) 22480
dropout_3 (Dropout) (None, 40) 0
dense_5 (Dense) (None, 1) 41
=================================================================
Total params: 22,521
Trainable params: 22,521
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 4000
dropout_4 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 160) 153760
dropout_5 (Dropout) (None, 128, 160) 0
max_pooling1d_2 (MaxPooling (None, 64, 160) 0
1D)
conv1d_3 (Conv1D) (None, 64, 64) 163904
dropout_6 (Dropout) (None, 64, 64) 0
max_pooling1d_3 (MaxPooling (None, 32, 64) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 4104
dropout_7 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 32) 3104
dropout_8 (Dropout) (None, 16, 32) 0
max_pooling1d_5 (MaxPooling (None, 8, 32) 0
1D)
conv1d_6 (Conv1D) (None, 8, 112) 28784
dropout_9 (Dropout) (None, 8, 112) 0
max_pooling1d_6 (MaxPooling (None, 4, 112) 0
1D)
flatten_1 (Flatten) (None, 448) 0
dense_6 (Dense) (None, 128) 57472
dropout_10 (Dropout) (None, 128) 0
dense_7 (Dense) (None, 1) 129
=================================================================
Total params: 415,257
Trainable params: 415,257
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 4000 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 160) 153760 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 160) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 160) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 64) 163904 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 64) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 64) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 4104 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_7[0][0]']
conv1d_5 (Conv1D) (None, 16, 32) 3104 ['max_pooling1d_4[0][0]']
dropout_8 (Dropout) (None, 16, 32) 0 ['conv1d_5[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 32) 0 ['dropout_8[0][0]']
conv1d_6 (Conv1D) (None, 8, 112) 28784 ['max_pooling1d_5[0][0]']
dropout_9 (Dropout) (None, 8, 112) 0 ['conv1d_6[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 112) 0 ['dropout_9[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
flatten_1 (Flatten) (None, 448) 0 ['max_pooling1d_6[0][0]']
dense_4 (Dense) (None, 40) 22480 ['epigenomic_data[0][0]']
dense_6 (Dense) (None, 128) 57472 ['flatten_1[0][0]']
dropout_3 (Dropout) (None, 40) 0 ['dense_4[0][0]']
dropout_10 (Dropout) (None, 128) 0 ['dense_6[0][0]']
concatenate (Concatenate) (None, 168) 0 ['dropout_3[0][0]',
'dropout_10[0][0]']
dense_8 (Dense) (None, 64) 10816 ['concatenate[0][0]']
dense_9 (Dense) (None, 1) 65 ['dense_8[0][0]']
==================================================================================================
Total params: 448,489
Trainable params: 448,489
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 17s 46ms/step - loss: 0.3826 - accuracy: 0.8150 - recall: 0.6371 - precision: 0.6428 - AUROC: 0.8719 - AUPRC: 0.6452 - f1_score: 0.6400 - balanced_accuracy: 0.7570 - specificity: 0.8768 - miss_rate: 0.3629 - fall_out: 0.1232 - mcc: 0.5155 - val_loss: 0.3443 - val_accuracy: 0.8382 - val_recall: 0.8119 - val_precision: 0.6491 - val_AUROC: 0.9019 - val_AUPRC: 0.6947 - val_f1_score: 0.7215 - val_balanced_accuracy: 0.8296 - val_specificity: 0.8473 - val_miss_rate: 0.1881 - val_fall_out: 0.1527 - val_mcc: 0.6170
Epoch 2/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3519 - accuracy: 0.8332 - recall: 0.7278 - precision: 0.6607 - AUROC: 0.8925 - AUPRC: 0.6842 - f1_score: 0.6926 - balanced_accuracy: 0.7989 - specificity: 0.8699 - miss_rate: 0.2722 - fall_out: 0.1301 - mcc: 0.5798 - val_loss: 0.3374 - val_accuracy: 0.8441 - val_recall: 0.7737 - val_precision: 0.6721 - val_AUROC: 0.9066 - val_AUPRC: 0.7146 - val_f1_score: 0.7193 - val_balanced_accuracy: 0.8212 - val_specificity: 0.8686 - val_miss_rate: 0.2263 - val_fall_out: 0.1314 - val_mcc: 0.6151
Epoch 3/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3411 - accuracy: 0.8384 - recall: 0.7370 - precision: 0.6699 - AUROC: 0.8987 - AUPRC: 0.6991 - f1_score: 0.7019 - balanced_accuracy: 0.8053 - specificity: 0.8737 - miss_rate: 0.2630 - fall_out: 0.1263 - mcc: 0.5926 - val_loss: 0.3291 - val_accuracy: 0.8459 - val_recall: 0.7545 - val_precision: 0.6823 - val_AUROC: 0.9090 - val_AUPRC: 0.7185 - val_f1_score: 0.7166 - val_balanced_accuracy: 0.8161 - val_specificity: 0.8777 - val_miss_rate: 0.2455 - val_fall_out: 0.1223 - val_mcc: 0.6126
Epoch 4/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3351 - accuracy: 0.8425 - recall: 0.7532 - precision: 0.6746 - AUROC: 0.9025 - AUPRC: 0.7068 - f1_score: 0.7117 - balanced_accuracy: 0.8134 - specificity: 0.8736 - miss_rate: 0.2468 - fall_out: 0.1264 - mcc: 0.6055 - val_loss: 0.3269 - val_accuracy: 0.8476 - val_recall: 0.7576 - val_precision: 0.6853 - val_AUROC: 0.9109 - val_AUPRC: 0.7293 - val_f1_score: 0.7197 - val_balanced_accuracy: 0.8183 - val_specificity: 0.8789 - val_miss_rate: 0.2424 - val_fall_out: 0.1211 - val_mcc: 0.6169
Epoch 5/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3303 - accuracy: 0.8452 - recall: 0.7558 - precision: 0.6801 - AUROC: 0.9061 - AUPRC: 0.7187 - f1_score: 0.7160 - balanced_accuracy: 0.8161 - specificity: 0.8763 - miss_rate: 0.2442 - fall_out: 0.1237 - mcc: 0.6116 - val_loss: 0.3307 - val_accuracy: 0.8454 - val_recall: 0.6539 - val_precision: 0.7211 - val_AUROC: 0.9115 - val_AUPRC: 0.7229 - val_f1_score: 0.6859 - val_balanced_accuracy: 0.7829 - val_specificity: 0.9120 - val_miss_rate: 0.3461 - val_fall_out: 0.0880 - val_mcc: 0.5849
Epoch 6/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3264 - accuracy: 0.8481 - recall: 0.7534 - precision: 0.6879 - AUROC: 0.9085 - AUPRC: 0.7283 - f1_score: 0.7191 - balanced_accuracy: 0.8172 - specificity: 0.8811 - miss_rate: 0.2466 - fall_out: 0.1189 - mcc: 0.6165 - val_loss: 0.3217 - val_accuracy: 0.8505 - val_recall: 0.7549 - val_precision: 0.6933 - val_AUROC: 0.9139 - val_AUPRC: 0.7325 - val_f1_score: 0.7228 - val_balanced_accuracy: 0.8194 - val_specificity: 0.8838 - val_miss_rate: 0.2451 - val_fall_out: 0.1162 - val_mcc: 0.6218
Epoch 7/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3216 - accuracy: 0.8503 - recall: 0.7451 - precision: 0.6963 - AUROC: 0.9116 - AUPRC: 0.7389 - f1_score: 0.7199 - balanced_accuracy: 0.8160 - specificity: 0.8869 - miss_rate: 0.2549 - fall_out: 0.1131 - mcc: 0.6186 - val_loss: 0.3176 - val_accuracy: 0.8544 - val_recall: 0.7349 - val_precision: 0.7109 - val_AUROC: 0.9157 - val_AUPRC: 0.7427 - val_f1_score: 0.7227 - val_balanced_accuracy: 0.8155 - val_specificity: 0.8960 - val_miss_rate: 0.2651 - val_fall_out: 0.1040 - val_mcc: 0.6242
Epoch 8/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3163 - accuracy: 0.8530 - recall: 0.7472 - precision: 0.7024 - AUROC: 0.9152 - AUPRC: 0.7551 - f1_score: 0.7241 - balanced_accuracy: 0.8185 - specificity: 0.8898 - miss_rate: 0.2528 - fall_out: 0.1102 - mcc: 0.6246 - val_loss: 0.3134 - val_accuracy: 0.8542 - val_recall: 0.7561 - val_precision: 0.7020 - val_AUROC: 0.9173 - val_AUPRC: 0.7530 - val_f1_score: 0.7280 - val_balanced_accuracy: 0.8222 - val_specificity: 0.8883 - val_miss_rate: 0.2439 - val_fall_out: 0.1117 - val_mcc: 0.6294
Epoch 9/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3098 - accuracy: 0.8580 - recall: 0.7465 - precision: 0.7156 - AUROC: 0.9190 - AUPRC: 0.7703 - f1_score: 0.7307 - balanced_accuracy: 0.8216 - specificity: 0.8968 - miss_rate: 0.2535 - fall_out: 0.1032 - mcc: 0.6346 - val_loss: 0.3152 - val_accuracy: 0.8521 - val_recall: 0.7202 - val_precision: 0.7108 - val_AUROC: 0.9166 - val_AUPRC: 0.7490 - val_f1_score: 0.7155 - val_balanced_accuracy: 0.8091 - val_specificity: 0.8980 - val_miss_rate: 0.2798 - val_fall_out: 0.1020 - val_mcc: 0.6156
Epoch 10/1000
313/313 [==============================] - 14s 44ms/step - loss: 0.3021 - accuracy: 0.8624 - recall: 0.7462 - precision: 0.7276 - AUROC: 0.9234 - AUPRC: 0.7879 - f1_score: 0.7368 - balanced_accuracy: 0.8245 - specificity: 0.9028 - miss_rate: 0.2538 - fall_out: 0.0972 - mcc: 0.6438 - val_loss: 0.3168 - val_accuracy: 0.8466 - val_recall: 0.6219 - val_precision: 0.7422 - val_AUROC: 0.9178 - val_AUPRC: 0.7548 - val_f1_score: 0.6767 - val_balanced_accuracy: 0.7734 - val_specificity: 0.9248 - val_miss_rate: 0.3781 - val_fall_out: 0.0752 - val_mcc: 0.5811
313/313 [==============================] - 7s 22ms/step - loss: 0.2799 - accuracy: 0.8793 - recall: 0.6906 - precision: 0.8136 - AUROC: 0.9409 - AUPRC: 0.8471 - f1_score: 0.7471 - balanced_accuracy: 0.8178 - specificity: 0.9450 - miss_rate: 0.3094 - fall_out: 0.0550 - mcc: 0.6724
79/79 [==============================] - 4s 46ms/step - loss: 0.3168 - accuracy: 0.8466 - recall: 0.6219 - precision: 0.7422 - AUROC: 0.9178 - AUPRC: 0.7548 - f1_score: 0.6767 - balanced_accuracy: 0.7734 - specificity: 0.9248 - miss_rate: 0.3781 - fall_out: 0.0752 - mcc: 0.5811
-- HOLDOUT 5 --
-- 3 Uncorrelated features: [Pearson+Spearman]
['SRSF9', 'ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['SRSF9', 'ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 18s 52ms/step - loss: 11.1934 - accuracy: 0.8083 - recall: 0.6023 - precision: 0.6357 - AUROC: 0.8650 - AUPRC: 0.6325 - f1_score: 0.6185 - balanced_accuracy: 0.7411 - specificity: 0.8799 - miss_rate: 0.3977 - fall_out: 0.1201 - mcc: 0.4909 - val_loss: 1.8926 - val_accuracy: 0.8364 - val_recall: 0.8123 - val_precision: 0.6456 - val_AUROC: 0.8995 - val_AUPRC: 0.7045 - val_f1_score: 0.7194 - val_balanced_accuracy: 0.8285 - val_specificity: 0.8448 - val_miss_rate: 0.1877 - val_fall_out: 0.1552 - val_mcc: 0.6140
Epoch 2/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.9354 - accuracy: 0.8323 - recall: 0.7354 - precision: 0.6562 - AUROC: 0.8919 - AUPRC: 0.6802 - f1_score: 0.6936 - balanced_accuracy: 0.8007 - specificity: 0.8660 - miss_rate: 0.2646 - fall_out: 0.1340 - mcc: 0.5804 - val_loss: 2.0217 - val_accuracy: 0.8416 - val_recall: 0.7974 - val_precision: 0.6599 - val_AUROC: 0.9028 - val_AUPRC: 0.7086 - val_f1_score: 0.7222 - val_balanced_accuracy: 0.8272 - val_specificity: 0.8570 - val_miss_rate: 0.2026 - val_fall_out: 0.1430 - val_mcc: 0.6181
Epoch 3/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.7472 - accuracy: 0.8375 - recall: 0.7566 - precision: 0.6622 - AUROC: 0.8978 - AUPRC: 0.6907 - f1_score: 0.7062 - balanced_accuracy: 0.8111 - specificity: 0.8657 - miss_rate: 0.2434 - fall_out: 0.1343 - mcc: 0.5972 - val_loss: 1.5183 - val_accuracy: 0.8436 - val_recall: 0.8086 - val_precision: 0.6612 - val_AUROC: 0.9042 - val_AUPRC: 0.7084 - val_f1_score: 0.7275 - val_balanced_accuracy: 0.8322 - val_specificity: 0.8558 - val_miss_rate: 0.1914 - val_fall_out: 0.1442 - val_mcc: 0.6256
Epoch 4/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.6250 - accuracy: 0.8418 - recall: 0.7628 - precision: 0.6700 - AUROC: 0.9014 - AUPRC: 0.6999 - f1_score: 0.7134 - balanced_accuracy: 0.8160 - specificity: 0.8693 - miss_rate: 0.2372 - fall_out: 0.1307 - mcc: 0.6072 - val_loss: 1.7933 - val_accuracy: 0.8428 - val_recall: 0.7964 - val_precision: 0.6626 - val_AUROC: 0.9056 - val_AUPRC: 0.7163 - val_f1_score: 0.7234 - val_balanced_accuracy: 0.8277 - val_specificity: 0.8589 - val_miss_rate: 0.2036 - val_fall_out: 0.1411 - val_mcc: 0.6199
Epoch 5/1000
313/313 [==============================] - 16s 51ms/step - loss: 1.5377 - accuracy: 0.8426 - recall: 0.7611 - precision: 0.6724 - AUROC: 0.9031 - AUPRC: 0.7056 - f1_score: 0.7140 - balanced_accuracy: 0.8160 - specificity: 0.8710 - miss_rate: 0.2389 - fall_out: 0.1290 - mcc: 0.6082 - val_loss: 1.2641 - val_accuracy: 0.8444 - val_recall: 0.7979 - val_precision: 0.6656 - val_AUROC: 0.9072 - val_AUPRC: 0.7237 - val_f1_score: 0.7258 - val_balanced_accuracy: 0.8292 - val_specificity: 0.8605 - val_miss_rate: 0.2021 - val_fall_out: 0.1395 - val_mcc: 0.6234
Epoch 6/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.4661 - accuracy: 0.8440 - recall: 0.7638 - precision: 0.6748 - AUROC: 0.9049 - AUPRC: 0.7093 - f1_score: 0.7165 - balanced_accuracy: 0.8178 - specificity: 0.8719 - miss_rate: 0.2362 - fall_out: 0.1281 - mcc: 0.6117 - val_loss: 1.7248 - val_accuracy: 0.8456 - val_recall: 0.7504 - val_precision: 0.6828 - val_AUROC: 0.9088 - val_AUPRC: 0.7279 - val_f1_score: 0.7150 - val_balanced_accuracy: 0.8146 - val_specificity: 0.8787 - val_miss_rate: 0.2496 - val_fall_out: 0.1213 - val_mcc: 0.6107
Epoch 7/1000
313/313 [==============================] - 16s 51ms/step - loss: 1.4083 - accuracy: 0.8462 - recall: 0.7683 - precision: 0.6784 - AUROC: 0.9072 - AUPRC: 0.7151 - f1_score: 0.7205 - balanced_accuracy: 0.8208 - specificity: 0.8732 - miss_rate: 0.2317 - fall_out: 0.1268 - mcc: 0.6173 - val_loss: 1.2072 - val_accuracy: 0.8455 - val_recall: 0.7250 - val_precision: 0.6915 - val_AUROC: 0.9091 - val_AUPRC: 0.7289 - val_f1_score: 0.7079 - val_balanced_accuracy: 0.8062 - val_specificity: 0.8874 - val_miss_rate: 0.2750 - val_fall_out: 0.1126 - val_mcc: 0.6033
Epoch 8/1000
313/313 [==============================] - 16s 51ms/step - loss: 1.3585 - accuracy: 0.8469 - recall: 0.7643 - precision: 0.6814 - AUROC: 0.9081 - AUPRC: 0.7184 - f1_score: 0.7205 - balanced_accuracy: 0.8200 - specificity: 0.8757 - miss_rate: 0.2357 - fall_out: 0.1243 - mcc: 0.6175 - val_loss: 1.5405 - val_accuracy: 0.8462 - val_recall: 0.7826 - val_precision: 0.6741 - val_AUROC: 0.9096 - val_AUPRC: 0.7309 - val_f1_score: 0.7243 - val_balanced_accuracy: 0.8255 - val_specificity: 0.8684 - val_miss_rate: 0.2174 - val_fall_out: 0.1316 - val_mcc: 0.6218
Epoch 9/1000
313/313 [==============================] - 16s 51ms/step - loss: 1.3170 - accuracy: 0.8485 - recall: 0.7712 - precision: 0.6828 - AUROC: 0.9096 - AUPRC: 0.7219 - f1_score: 0.7243 - balanced_accuracy: 0.8233 - specificity: 0.8753 - miss_rate: 0.2288 - fall_out: 0.1247 - mcc: 0.6225 - val_loss: 1.1065 - val_accuracy: 0.8469 - val_recall: 0.7621 - val_precision: 0.6822 - val_AUROC: 0.9106 - val_AUPRC: 0.7318 - val_f1_score: 0.7199 - val_balanced_accuracy: 0.8193 - val_specificity: 0.8765 - val_miss_rate: 0.2379 - val_fall_out: 0.1235 - val_mcc: 0.6168
Epoch 10/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.2780 - accuracy: 0.8489 - recall: 0.7726 - precision: 0.6833 - AUROC: 0.9116 - AUPRC: 0.7299 - f1_score: 0.7252 - balanced_accuracy: 0.8240 - specificity: 0.8754 - miss_rate: 0.2274 - fall_out: 0.1246 - mcc: 0.6237 - val_loss: 1.3628 - val_accuracy: 0.8461 - val_recall: 0.7491 - val_precision: 0.6846 - val_AUROC: 0.9113 - val_AUPRC: 0.7358 - val_f1_score: 0.7154 - val_balanced_accuracy: 0.8145 - val_specificity: 0.8799 - val_miss_rate: 0.2509 - val_fall_out: 0.1201 - val_mcc: 0.6114
Epoch 11/1000
313/313 [==============================] - 16s 50ms/step - loss: 1.2477 - accuracy: 0.8497 - recall: 0.7731 - precision: 0.6851 - AUROC: 0.9117 - AUPRC: 0.7275 - f1_score: 0.7264 - balanced_accuracy: 0.8247 - specificity: 0.8763 - miss_rate: 0.2269 - fall_out: 0.1237 - mcc: 0.6255 - val_loss: 1.1293 - val_accuracy: 0.8480 - val_recall: 0.7466 - val_precision: 0.6900 - val_AUROC: 0.9115 - val_AUPRC: 0.7362 - val_f1_score: 0.7171 - val_balanced_accuracy: 0.8149 - val_specificity: 0.8833 - val_miss_rate: 0.2534 - val_fall_out: 0.1167 - val_mcc: 0.6143
313/313 [==============================] - 8s 25ms/step - loss: 1.1151 - accuracy: 0.8574 - recall: 0.7708 - precision: 0.7045 - AUROC: 0.9207 - AUPRC: 0.7531 - f1_score: 0.7361 - balanced_accuracy: 0.8291 - specificity: 0.8875 - miss_rate: 0.2292 - fall_out: 0.1125 - mcc: 0.6399
79/79 [==============================] - 4s 50ms/step - loss: 1.1293 - accuracy: 0.8480 - recall: 0.7466 - precision: 0.6900 - AUROC: 0.9115 - AUPRC: 0.7362 - f1_score: 0.7171 - balanced_accuracy: 0.8149 - specificity: 0.8833 - miss_rate: 0.2534 - fall_out: 0.1167 - mcc: 0.6143
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_4\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_4\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 104, 'dropout_0': 0.3, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 8, 'dropout_2': 0.5, 'units_3': 136, 'dropout_3': 0.5, 'units_4': 104, 'dropout_4': 0.5}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_4\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_4\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 16, 'conv_dropout_0': 0.0, 'dense_depth': 1, 'units_0': 16, 'dense_dropout_0': 0.3, 'filters_1': 160, 'kernel_size_1': 16, 'conv_dropout_1': 0.0, 'filters_2': 160, 'kernel_size_2': 16, 'conv_dropout_2': 0.2, 'units_1': 128, 'dense_dropout_1': 0.5, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.1, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.0, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['SRSF9', 'ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 104) 58448
dropout_3 (Dropout) (None, 104) 0
dense_5 (Dense) (None, 1) 105
=================================================================
Total params: 58,553
Trainable params: 58,553
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 10400
dropout_4 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 160) 409760
dropout_5 (Dropout) (None, 128, 160) 0
max_pooling1d_2 (MaxPooling (None, 64, 160) 0
1D)
conv1d_3 (Conv1D) (None, 64, 160) 409760
dropout_6 (Dropout) (None, 64, 160) 0
max_pooling1d_3 (MaxPooling (None, 32, 160) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 7688
dropout_7 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 8) 392
dropout_8 (Dropout) (None, 16, 8) 0
max_pooling1d_5 (MaxPooling (None, 8, 8) 0
1D)
conv1d_6 (Conv1D) (None, 8, 8) 392
dropout_9 (Dropout) (None, 8, 8) 0
max_pooling1d_6 (MaxPooling (None, 4, 8) 0
1D)
flatten_1 (Flatten) (None, 32) 0
dense_6 (Dense) (None, 16) 528
dropout_10 (Dropout) (None, 16) 0
dense_7 (Dense) (None, 1) 17
=================================================================
Total params: 838,937
Trainable params: 838,937
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 10400 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 160) 409760 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 160) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 160) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 160) 409760 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 160) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 160) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 7688 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_7[0][0]']
conv1d_5 (Conv1D) (None, 16, 8) 392 ['max_pooling1d_4[0][0]']
dropout_8 (Dropout) (None, 16, 8) 0 ['conv1d_5[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 8) 0 ['dropout_8[0][0]']
conv1d_6 (Conv1D) (None, 8, 8) 392 ['max_pooling1d_5[0][0]']
dropout_9 (Dropout) (None, 8, 8) 0 ['conv1d_6[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 8) 0 ['dropout_9[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
flatten_1 (Flatten) (None, 32) 0 ['max_pooling1d_6[0][0]']
dense_4 (Dense) (None, 104) 58448 ['epigenomic_data[0][0]']
dense_6 (Dense) (None, 16) 528 ['flatten_1[0][0]']
dropout_3 (Dropout) (None, 104) 0 ['dense_4[0][0]']
dropout_10 (Dropout) (None, 16) 0 ['dense_6[0][0]']
concatenate (Concatenate) (None, 120) 0 ['dropout_3[0][0]',
'dropout_10[0][0]']
dense_8 (Dense) (None, 64) 7744 ['concatenate[0][0]']
dense_9 (Dense) (None, 1) 65 ['dense_8[0][0]']
==================================================================================================
Total params: 905,177
Trainable params: 905,177
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 20s 53ms/step - loss: 0.3712 - accuracy: 0.8248 - recall: 0.6767 - precision: 0.6556 - AUROC: 0.8822 - AUPRC: 0.6628 - f1_score: 0.6660 - balanced_accuracy: 0.7765 - specificity: 0.8763 - miss_rate: 0.3233 - fall_out: 0.1237 - mcc: 0.5474 - val_loss: 0.3398 - val_accuracy: 0.8414 - val_recall: 0.7326 - val_precision: 0.6786 - val_AUROC: 0.9004 - val_AUPRC: 0.7043 - val_f1_score: 0.7046 - val_balanced_accuracy: 0.8059 - val_specificity: 0.8793 - val_miss_rate: 0.2674 - val_fall_out: 0.1207 - val_mcc: 0.5972
Epoch 2/1000
313/313 [==============================] - 16s 49ms/step - loss: 0.3393 - accuracy: 0.8402 - recall: 0.7335 - precision: 0.6753 - AUROC: 0.9006 - AUPRC: 0.7011 - f1_score: 0.7032 - balanced_accuracy: 0.8054 - specificity: 0.8773 - miss_rate: 0.2665 - fall_out: 0.1227 - mcc: 0.5950 - val_loss: 0.3350 - val_accuracy: 0.8425 - val_recall: 0.7665 - val_precision: 0.6705 - val_AUROC: 0.9032 - val_AUPRC: 0.7144 - val_f1_score: 0.7153 - val_balanced_accuracy: 0.8177 - val_specificity: 0.8689 - val_miss_rate: 0.2335 - val_fall_out: 0.1311 - val_mcc: 0.6097
Epoch 3/1000
313/313 [==============================] - 16s 50ms/step - loss: 0.3303 - accuracy: 0.8454 - recall: 0.7449 - precision: 0.6842 - AUROC: 0.9059 - AUPRC: 0.7173 - f1_score: 0.7132 - balanced_accuracy: 0.8126 - specificity: 0.8804 - miss_rate: 0.2551 - fall_out: 0.1196 - mcc: 0.6087 - val_loss: 0.3291 - val_accuracy: 0.8444 - val_recall: 0.7444 - val_precision: 0.6819 - val_AUROC: 0.9072 - val_AUPRC: 0.7233 - val_f1_score: 0.7118 - val_balanced_accuracy: 0.8118 - val_specificity: 0.8791 - val_miss_rate: 0.2556 - val_fall_out: 0.1209 - val_mcc: 0.6066
Epoch 4/1000
313/313 [==============================] - 16s 49ms/step - loss: 0.3235 - accuracy: 0.8486 - recall: 0.7461 - precision: 0.6916 - AUROC: 0.9105 - AUPRC: 0.7321 - f1_score: 0.7178 - balanced_accuracy: 0.8152 - specificity: 0.8842 - miss_rate: 0.2539 - fall_out: 0.1158 - mcc: 0.6154 - val_loss: 0.3270 - val_accuracy: 0.8477 - val_recall: 0.7338 - val_precision: 0.6939 - val_AUROC: 0.9093 - val_AUPRC: 0.7288 - val_f1_score: 0.7133 - val_balanced_accuracy: 0.8106 - val_specificity: 0.8874 - val_miss_rate: 0.2662 - val_fall_out: 0.1126 - val_mcc: 0.6102
Epoch 5/1000
313/313 [==============================] - 16s 49ms/step - loss: 0.3178 - accuracy: 0.8535 - recall: 0.7527 - precision: 0.7014 - AUROC: 0.9140 - AUPRC: 0.7416 - f1_score: 0.7262 - balanced_accuracy: 0.8206 - specificity: 0.8885 - miss_rate: 0.2473 - fall_out: 0.1115 - mcc: 0.6270 - val_loss: 0.3216 - val_accuracy: 0.8496 - val_recall: 0.7526 - val_precision: 0.6919 - val_AUROC: 0.9115 - val_AUPRC: 0.7385 - val_f1_score: 0.7210 - val_balanced_accuracy: 0.8180 - val_specificity: 0.8834 - val_miss_rate: 0.2474 - val_fall_out: 0.1166 - val_mcc: 0.6193
Epoch 6/1000
313/313 [==============================] - 16s 49ms/step - loss: 0.3103 - accuracy: 0.8558 - recall: 0.7454 - precision: 0.7101 - AUROC: 0.9186 - AUPRC: 0.7604 - f1_score: 0.7274 - balanced_accuracy: 0.8198 - specificity: 0.8941 - miss_rate: 0.2546 - fall_out: 0.1059 - mcc: 0.6297 - val_loss: 0.3244 - val_accuracy: 0.8462 - val_recall: 0.6709 - val_precision: 0.7156 - val_AUROC: 0.9115 - val_AUPRC: 0.7434 - val_f1_score: 0.6926 - val_balanced_accuracy: 0.7891 - val_specificity: 0.9072 - val_miss_rate: 0.3291 - val_fall_out: 0.0928 - val_mcc: 0.5907
Epoch 7/1000
313/313 [==============================] - 16s 50ms/step - loss: 0.3015 - accuracy: 0.8599 - recall: 0.7458 - precision: 0.7210 - AUROC: 0.9236 - AUPRC: 0.7783 - f1_score: 0.7332 - balanced_accuracy: 0.8227 - specificity: 0.8996 - miss_rate: 0.2542 - fall_out: 0.1004 - mcc: 0.6384 - val_loss: 0.3218 - val_accuracy: 0.8513 - val_recall: 0.7196 - val_precision: 0.7089 - val_AUROC: 0.9128 - val_AUPRC: 0.7403 - val_f1_score: 0.7142 - val_balanced_accuracy: 0.8084 - val_specificity: 0.8972 - val_miss_rate: 0.2804 - val_fall_out: 0.1028 - val_mcc: 0.6138
313/313 [==============================] - 8s 24ms/step - loss: 0.2917 - accuracy: 0.8706 - recall: 0.7613 - precision: 0.7435 - AUROC: 0.9325 - AUPRC: 0.8048 - f1_score: 0.7523 - balanced_accuracy: 0.8350 - specificity: 0.9086 - miss_rate: 0.2387 - fall_out: 0.0914 - mcc: 0.6648
79/79 [==============================] - 4s 48ms/step - loss: 0.3218 - accuracy: 0.8513 - recall: 0.7196 - precision: 0.7089 - AUROC: 0.9128 - AUPRC: 0.7403 - f1_score: 0.7142 - balanced_accuracy: 0.8084 - specificity: 0.8972 - miss_rate: 0.2804 - fall_out: 0.1028 - mcc: 0.6138
-- HOLDOUT 6 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 20s 53ms/step - loss: 11.1539 - accuracy: 0.8015 - recall: 0.5578 - precision: 0.6305 - AUROC: 0.8585 - AUPRC: 0.6205 - f1_score: 0.5919 - balanced_accuracy: 0.7220 - specificity: 0.8863 - miss_rate: 0.4422 - fall_out: 0.1137 - mcc: 0.4629 - val_loss: 2.2253 - val_accuracy: 0.8371 - val_recall: 0.7173 - val_precision: 0.6730 - val_AUROC: 0.8989 - val_AUPRC: 0.6972 - val_f1_score: 0.6945 - val_balanced_accuracy: 0.7980 - val_specificity: 0.8787 - val_miss_rate: 0.2827 - val_fall_out: 0.1213 - val_mcc: 0.5841
Epoch 2/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.9444 - accuracy: 0.8306 - recall: 0.7141 - precision: 0.6586 - AUROC: 0.8899 - AUPRC: 0.6728 - f1_score: 0.6852 - balanced_accuracy: 0.7926 - specificity: 0.8712 - miss_rate: 0.2859 - fall_out: 0.1288 - mcc: 0.5705 - val_loss: 1.6294 - val_accuracy: 0.8421 - val_recall: 0.7177 - val_precision: 0.6854 - val_AUROC: 0.9034 - val_AUPRC: 0.7115 - val_f1_score: 0.7011 - val_balanced_accuracy: 0.8015 - val_specificity: 0.8854 - val_miss_rate: 0.2823 - val_fall_out: 0.1146 - val_mcc: 0.5942
Epoch 3/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.7480 - accuracy: 0.8357 - recall: 0.7426 - precision: 0.6619 - AUROC: 0.8959 - AUPRC: 0.6870 - f1_score: 0.6999 - balanced_accuracy: 0.8053 - specificity: 0.8680 - miss_rate: 0.2574 - fall_out: 0.1320 - mcc: 0.5891 - val_loss: 1.8479 - val_accuracy: 0.8422 - val_recall: 0.7801 - val_precision: 0.6658 - val_AUROC: 0.9039 - val_AUPRC: 0.7172 - val_f1_score: 0.7185 - val_balanced_accuracy: 0.8219 - val_specificity: 0.8638 - val_miss_rate: 0.2199 - val_fall_out: 0.1362 - val_mcc: 0.6134
Epoch 4/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.6235 - accuracy: 0.8384 - recall: 0.7468 - precision: 0.6671 - AUROC: 0.8992 - AUPRC: 0.6949 - f1_score: 0.7047 - balanced_accuracy: 0.8086 - specificity: 0.8703 - miss_rate: 0.2532 - fall_out: 0.1297 - mcc: 0.5958 - val_loss: 1.3938 - val_accuracy: 0.8441 - val_recall: 0.7683 - val_precision: 0.6737 - val_AUROC: 0.9062 - val_AUPRC: 0.7143 - val_f1_score: 0.7179 - val_balanced_accuracy: 0.8194 - val_specificity: 0.8705 - val_miss_rate: 0.2317 - val_fall_out: 0.1295 - val_mcc: 0.6133
Epoch 5/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.5324 - accuracy: 0.8418 - recall: 0.7606 - precision: 0.6706 - AUROC: 0.9024 - AUPRC: 0.7039 - f1_score: 0.7128 - balanced_accuracy: 0.8153 - specificity: 0.8700 - miss_rate: 0.2394 - fall_out: 0.1300 - mcc: 0.6064 - val_loss: 1.6686 - val_accuracy: 0.8443 - val_recall: 0.7563 - val_precision: 0.6779 - val_AUROC: 0.9076 - val_AUPRC: 0.7222 - val_f1_score: 0.7149 - val_balanced_accuracy: 0.8156 - val_specificity: 0.8750 - val_miss_rate: 0.2437 - val_fall_out: 0.1250 - val_mcc: 0.6100
Epoch 6/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.4620 - accuracy: 0.8443 - recall: 0.7510 - precision: 0.6796 - AUROC: 0.9056 - AUPRC: 0.7113 - f1_score: 0.7135 - balanced_accuracy: 0.8139 - specificity: 0.8768 - miss_rate: 0.2490 - fall_out: 0.1232 - mcc: 0.6084 - val_loss: 1.2760 - val_accuracy: 0.8451 - val_recall: 0.7283 - val_precision: 0.6893 - val_AUROC: 0.9086 - val_AUPRC: 0.7262 - val_f1_score: 0.7083 - val_balanced_accuracy: 0.8070 - val_specificity: 0.8858 - val_miss_rate: 0.2717 - val_fall_out: 0.1142 - val_mcc: 0.6034
Epoch 7/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.4068 - accuracy: 0.8455 - recall: 0.7526 - precision: 0.6819 - AUROC: 0.9060 - AUPRC: 0.7136 - f1_score: 0.7155 - balanced_accuracy: 0.8152 - specificity: 0.8779 - miss_rate: 0.2474 - fall_out: 0.1221 - mcc: 0.6112 - val_loss: 1.5209 - val_accuracy: 0.8456 - val_recall: 0.7497 - val_precision: 0.6832 - val_AUROC: 0.9093 - val_AUPRC: 0.7280 - val_f1_score: 0.7149 - val_balanced_accuracy: 0.8143 - val_specificity: 0.8790 - val_miss_rate: 0.2503 - val_fall_out: 0.1210 - val_mcc: 0.6106
Epoch 8/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.3597 - accuracy: 0.8468 - recall: 0.7474 - precision: 0.6867 - AUROC: 0.9075 - AUPRC: 0.7177 - f1_score: 0.7157 - balanced_accuracy: 0.8144 - specificity: 0.8813 - miss_rate: 0.2526 - fall_out: 0.1187 - mcc: 0.6121 - val_loss: 1.2598 - val_accuracy: 0.8463 - val_recall: 0.7281 - val_precision: 0.6924 - val_AUROC: 0.9099 - val_AUPRC: 0.7302 - val_f1_score: 0.7098 - val_balanced_accuracy: 0.8078 - val_specificity: 0.8874 - val_miss_rate: 0.2719 - val_fall_out: 0.1126 - val_mcc: 0.6058
Epoch 9/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.3170 - accuracy: 0.8476 - recall: 0.7572 - precision: 0.6853 - AUROC: 0.9089 - AUPRC: 0.7208 - f1_score: 0.7194 - balanced_accuracy: 0.8181 - specificity: 0.8790 - miss_rate: 0.2428 - fall_out: 0.1210 - mcc: 0.6166 - val_loss: 1.4301 - val_accuracy: 0.8454 - val_recall: 0.7778 - val_precision: 0.6737 - val_AUROC: 0.9104 - val_AUPRC: 0.7340 - val_f1_score: 0.7220 - val_balanced_accuracy: 0.8233 - val_specificity: 0.8689 - val_miss_rate: 0.2222 - val_fall_out: 0.1311 - val_mcc: 0.6187
Epoch 10/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.2816 - accuracy: 0.8488 - recall: 0.7632 - precision: 0.6862 - AUROC: 0.9097 - AUPRC: 0.7213 - f1_score: 0.7226 - balanced_accuracy: 0.8209 - specificity: 0.8786 - miss_rate: 0.2368 - fall_out: 0.1214 - mcc: 0.6207 - val_loss: 1.1709 - val_accuracy: 0.8471 - val_recall: 0.7347 - val_precision: 0.6921 - val_AUROC: 0.9110 - val_AUPRC: 0.7348 - val_f1_score: 0.7128 - val_balanced_accuracy: 0.8105 - val_specificity: 0.8862 - val_miss_rate: 0.2653 - val_fall_out: 0.1138 - val_mcc: 0.6092
Epoch 11/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.2474 - accuracy: 0.8504 - recall: 0.7630 - precision: 0.6901 - AUROC: 0.9118 - AUPRC: 0.7303 - f1_score: 0.7247 - balanced_accuracy: 0.8219 - specificity: 0.8808 - miss_rate: 0.2370 - fall_out: 0.1192 - mcc: 0.6238 - val_loss: 1.2924 - val_accuracy: 0.8484 - val_recall: 0.7535 - val_precision: 0.6885 - val_AUROC: 0.9115 - val_AUPRC: 0.7375 - val_f1_score: 0.7196 - val_balanced_accuracy: 0.8175 - val_specificity: 0.8814 - val_miss_rate: 0.2465 - val_fall_out: 0.1186 - val_mcc: 0.6171
Epoch 12/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.2150 - accuracy: 0.8518 - recall: 0.7683 - precision: 0.6916 - AUROC: 0.9130 - AUPRC: 0.7325 - f1_score: 0.7279 - balanced_accuracy: 0.8246 - specificity: 0.8808 - miss_rate: 0.2317 - fall_out: 0.1192 - mcc: 0.6281 - val_loss: 1.1651 - val_accuracy: 0.8504 - val_recall: 0.7353 - val_precision: 0.7001 - val_AUROC: 0.9118 - val_AUPRC: 0.7360 - val_f1_score: 0.7173 - val_balanced_accuracy: 0.8129 - val_specificity: 0.8904 - val_miss_rate: 0.2647 - val_fall_out: 0.1096 - val_mcc: 0.6160
Epoch 13/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.1853 - accuracy: 0.8510 - recall: 0.7704 - precision: 0.6891 - AUROC: 0.9137 - AUPRC: 0.7336 - f1_score: 0.7275 - balanced_accuracy: 0.8248 - specificity: 0.8791 - miss_rate: 0.2296 - fall_out: 0.1209 - mcc: 0.6273 - val_loss: 1.1904 - val_accuracy: 0.8507 - val_recall: 0.7551 - val_precision: 0.6937 - val_AUROC: 0.9123 - val_AUPRC: 0.7389 - val_f1_score: 0.7231 - val_balanced_accuracy: 0.8195 - val_specificity: 0.8840 - val_miss_rate: 0.2449 - val_fall_out: 0.1160 - val_mcc: 0.6222
Epoch 14/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.1546 - accuracy: 0.8546 - recall: 0.7820 - precision: 0.6938 - AUROC: 0.9154 - AUPRC: 0.7362 - f1_score: 0.7352 - balanced_accuracy: 0.8309 - specificity: 0.8799 - miss_rate: 0.2180 - fall_out: 0.1201 - mcc: 0.6377 - val_loss: 1.2021 - val_accuracy: 0.8497 - val_recall: 0.7708 - val_precision: 0.6859 - val_AUROC: 0.9125 - val_AUPRC: 0.7394 - val_f1_score: 0.7259 - val_balanced_accuracy: 0.8240 - val_specificity: 0.8772 - val_miss_rate: 0.2292 - val_fall_out: 0.1228 - val_mcc: 0.6249
313/313 [==============================] - 8s 26ms/step - loss: 1.1825 - accuracy: 0.8628 - recall: 0.7866 - precision: 0.7119 - AUROC: 0.9254 - AUPRC: 0.7638 - f1_score: 0.7474 - balanced_accuracy: 0.8379 - specificity: 0.8893 - miss_rate: 0.2134 - fall_out: 0.1107 - mcc: 0.6550
79/79 [==============================] - 4s 52ms/step - loss: 1.2021 - accuracy: 0.8497 - recall: 0.7708 - precision: 0.6859 - AUROC: 0.9125 - AUPRC: 0.7394 - f1_score: 0.7259 - balanced_accuracy: 0.8240 - specificity: 0.8772 - miss_rate: 0.2292 - fall_out: 0.1228 - mcc: 0.6249
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_5\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_5\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 40, 'dropout_1': 0.5, 'units_2': 40, 'dropout_2': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_5\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_5\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 6, 'conv_dropout_0': 0.2, 'dense_depth': 1, 'units_0': 128, 'dense_dropout_0': 0.3, 'filters_1': 8, 'kernel_size_1': 16, 'conv_dropout_1': 0.2, 'units_1': 16, 'dense_dropout_1': 0.5, 'filters_2': 8, 'kernel_size_2': 12, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.1, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.0, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 1) 137
=================================================================
Total params: 76,569
Trainable params: 76,569
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 4000
dropout_4 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 8) 20488
dropout_5 (Dropout) (None, 128, 8) 0
max_pooling1d_2 (MaxPooling (None, 64, 8) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 776
dropout_6 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 392
dropout_7 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 8) 392
dropout_8 (Dropout) (None, 16, 8) 0
max_pooling1d_5 (MaxPooling (None, 8, 8) 0
1D)
conv1d_6 (Conv1D) (None, 8, 8) 392
dropout_9 (Dropout) (None, 8, 8) 0
max_pooling1d_6 (MaxPooling (None, 4, 8) 0
1D)
flatten_1 (Flatten) (None, 32) 0
dense_6 (Dense) (None, 128) 4224
dropout_10 (Dropout) (None, 128) 0
dense_7 (Dense) (None, 1) 129
=================================================================
Total params: 30,793
Trainable params: 30,793
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 4000 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 8) 20488 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 8) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 8) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 776 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 392 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_7[0][0]']
conv1d_5 (Conv1D) (None, 16, 8) 392 ['max_pooling1d_4[0][0]']
dropout_8 (Dropout) (None, 16, 8) 0 ['conv1d_5[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 8) 0 ['dropout_8[0][0]']
conv1d_6 (Conv1D) (None, 8, 8) 392 ['max_pooling1d_5[0][0]']
dropout_9 (Dropout) (None, 8, 8) 0 ['conv1d_6[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 8) 0 ['dropout_9[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
flatten_1 (Flatten) (None, 32) 0 ['max_pooling1d_6[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
dense_6 (Dense) (None, 128) 4224 ['flatten_1[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_10 (Dropout) (None, 128) 0 ['dense_6[0][0]']
concatenate (Concatenate) (None, 264) 0 ['dropout_3[0][0]',
'dropout_10[0][0]']
dense_8 (Dense) (None, 64) 16960 ['concatenate[0][0]']
dense_9 (Dense) (None, 1) 65 ['dense_8[0][0]']
==================================================================================================
Total params: 124,121
Trainable params: 124,121
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 18s 47ms/step - loss: 0.3819 - accuracy: 0.8182 - recall: 0.6457 - precision: 0.6485 - AUROC: 0.8757 - AUPRC: 0.6483 - f1_score: 0.6471 - balanced_accuracy: 0.7619 - specificity: 0.8782 - miss_rate: 0.3543 - fall_out: 0.1218 - mcc: 0.5246 - val_loss: 0.3454 - val_accuracy: 0.8346 - val_recall: 0.8216 - val_precision: 0.6398 - val_AUROC: 0.8989 - val_AUPRC: 0.7020 - val_f1_score: 0.7194 - val_balanced_accuracy: 0.8303 - val_specificity: 0.8391 - val_miss_rate: 0.1784 - val_fall_out: 0.1609 - val_mcc: 0.6142
Epoch 2/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.3457 - accuracy: 0.8359 - recall: 0.7272 - precision: 0.6670 - AUROC: 0.8960 - AUPRC: 0.6887 - f1_score: 0.6958 - balanced_accuracy: 0.8004 - specificity: 0.8737 - miss_rate: 0.2728 - fall_out: 0.1263 - mcc: 0.5847 - val_loss: 0.3380 - val_accuracy: 0.8383 - val_recall: 0.8096 - val_precision: 0.6499 - val_AUROC: 0.9035 - val_AUPRC: 0.7049 - val_f1_score: 0.7210 - val_balanced_accuracy: 0.8289 - val_specificity: 0.8482 - val_miss_rate: 0.1904 - val_fall_out: 0.1518 - val_mcc: 0.6163
Epoch 3/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.3372 - accuracy: 0.8406 - recall: 0.7409 - precision: 0.6740 - AUROC: 0.9010 - AUPRC: 0.7013 - f1_score: 0.7059 - balanced_accuracy: 0.8081 - specificity: 0.8753 - miss_rate: 0.2591 - fall_out: 0.1247 - mcc: 0.5982 - val_loss: 0.3310 - val_accuracy: 0.8429 - val_recall: 0.7772 - val_precision: 0.6682 - val_AUROC: 0.9061 - val_AUPRC: 0.7178 - val_f1_score: 0.7186 - val_balanced_accuracy: 0.8215 - val_specificity: 0.8657 - val_miss_rate: 0.2228 - val_fall_out: 0.1343 - val_mcc: 0.6138
Epoch 4/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.3303 - accuracy: 0.8440 - recall: 0.7476 - precision: 0.6799 - AUROC: 0.9053 - AUPRC: 0.7145 - f1_score: 0.7121 - balanced_accuracy: 0.8126 - specificity: 0.8775 - miss_rate: 0.2524 - fall_out: 0.1225 - mcc: 0.6067 - val_loss: 0.3301 - val_accuracy: 0.8426 - val_recall: 0.7921 - val_precision: 0.6635 - val_AUROC: 0.9079 - val_AUPRC: 0.7220 - val_f1_score: 0.7221 - val_balanced_accuracy: 0.8262 - val_specificity: 0.8602 - val_miss_rate: 0.2079 - val_fall_out: 0.1398 - val_mcc: 0.6182
Epoch 5/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3261 - accuracy: 0.8459 - recall: 0.7523 - precision: 0.6829 - AUROC: 0.9081 - AUPRC: 0.7204 - f1_score: 0.7159 - balanced_accuracy: 0.8154 - specificity: 0.8785 - miss_rate: 0.2477 - fall_out: 0.1215 - mcc: 0.6119 - val_loss: 0.3251 - val_accuracy: 0.8494 - val_recall: 0.7776 - val_precision: 0.6829 - val_AUROC: 0.9106 - val_AUPRC: 0.7349 - val_f1_score: 0.7272 - val_balanced_accuracy: 0.8260 - val_specificity: 0.8744 - val_miss_rate: 0.2224 - val_fall_out: 0.1256 - val_mcc: 0.6263
Epoch 6/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.3215 - accuracy: 0.8495 - recall: 0.7596 - precision: 0.6891 - AUROC: 0.9112 - AUPRC: 0.7317 - f1_score: 0.7226 - balanced_accuracy: 0.8202 - specificity: 0.8808 - miss_rate: 0.2404 - fall_out: 0.1192 - mcc: 0.6210 - val_loss: 0.3231 - val_accuracy: 0.8479 - val_recall: 0.7553 - val_precision: 0.6867 - val_AUROC: 0.9119 - val_AUPRC: 0.7390 - val_f1_score: 0.7194 - val_balanced_accuracy: 0.8177 - val_specificity: 0.8801 - val_miss_rate: 0.2447 - val_fall_out: 0.1199 - val_mcc: 0.6167
Epoch 7/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.3173 - accuracy: 0.8511 - recall: 0.7552 - precision: 0.6946 - AUROC: 0.9137 - AUPRC: 0.7396 - f1_score: 0.7237 - balanced_accuracy: 0.8198 - specificity: 0.8845 - miss_rate: 0.2448 - fall_out: 0.1155 - mcc: 0.6230 - val_loss: 0.3196 - val_accuracy: 0.8502 - val_recall: 0.7733 - val_precision: 0.6863 - val_AUROC: 0.9133 - val_AUPRC: 0.7444 - val_f1_score: 0.7272 - val_balanced_accuracy: 0.8252 - val_specificity: 0.8770 - val_miss_rate: 0.2267 - val_fall_out: 0.1230 - val_mcc: 0.6266
Epoch 8/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.3133 - accuracy: 0.8541 - recall: 0.7561 - precision: 0.7017 - AUROC: 0.9163 - AUPRC: 0.7492 - f1_score: 0.7279 - balanced_accuracy: 0.8221 - specificity: 0.8881 - miss_rate: 0.2439 - fall_out: 0.1119 - mcc: 0.6292 - val_loss: 0.3171 - val_accuracy: 0.8509 - val_recall: 0.7803 - val_precision: 0.6855 - val_AUROC: 0.9155 - val_AUPRC: 0.7544 - val_f1_score: 0.7298 - val_balanced_accuracy: 0.8279 - val_specificity: 0.8754 - val_miss_rate: 0.2197 - val_fall_out: 0.1246 - val_mcc: 0.6300
Epoch 9/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3095 - accuracy: 0.8557 - recall: 0.7579 - precision: 0.7050 - AUROC: 0.9187 - AUPRC: 0.7569 - f1_score: 0.7305 - balanced_accuracy: 0.8238 - specificity: 0.8897 - miss_rate: 0.2421 - fall_out: 0.1103 - mcc: 0.6329 - val_loss: 0.3172 - val_accuracy: 0.8493 - val_recall: 0.7999 - val_precision: 0.6759 - val_AUROC: 0.9152 - val_AUPRC: 0.7527 - val_f1_score: 0.7327 - val_balanced_accuracy: 0.8332 - val_specificity: 0.8665 - val_miss_rate: 0.2001 - val_fall_out: 0.1335 - val_mcc: 0.6331
Epoch 10/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.3051 - accuracy: 0.8580 - recall: 0.7609 - precision: 0.7098 - AUROC: 0.9211 - AUPRC: 0.7673 - f1_score: 0.7345 - balanced_accuracy: 0.8263 - specificity: 0.8918 - miss_rate: 0.2391 - fall_out: 0.1082 - mcc: 0.6384 - val_loss: 0.3140 - val_accuracy: 0.8538 - val_recall: 0.7516 - val_precision: 0.7028 - val_AUROC: 0.9161 - val_AUPRC: 0.7553 - val_f1_score: 0.7264 - val_balanced_accuracy: 0.8205 - val_specificity: 0.8894 - val_miss_rate: 0.2484 - val_fall_out: 0.1106 - val_mcc: 0.6275
Epoch 11/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3007 - accuracy: 0.8603 - recall: 0.7593 - precision: 0.7164 - AUROC: 0.9238 - AUPRC: 0.7764 - f1_score: 0.7372 - balanced_accuracy: 0.8274 - specificity: 0.8954 - miss_rate: 0.2407 - fall_out: 0.1046 - mcc: 0.6427 - val_loss: 0.3115 - val_accuracy: 0.8529 - val_recall: 0.7813 - val_precision: 0.6899 - val_AUROC: 0.9179 - val_AUPRC: 0.7639 - val_f1_score: 0.7327 - val_balanced_accuracy: 0.8295 - val_specificity: 0.8778 - val_miss_rate: 0.2187 - val_fall_out: 0.1222 - val_mcc: 0.6341
Epoch 12/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2960 - accuracy: 0.8631 - recall: 0.7651 - precision: 0.7215 - AUROC: 0.9263 - AUPRC: 0.7833 - f1_score: 0.7427 - balanced_accuracy: 0.8312 - specificity: 0.8972 - miss_rate: 0.2349 - fall_out: 0.1028 - mcc: 0.6501 - val_loss: 0.3140 - val_accuracy: 0.8522 - val_recall: 0.7937 - val_precision: 0.6842 - val_AUROC: 0.9169 - val_AUPRC: 0.7589 - val_f1_score: 0.7349 - val_balanced_accuracy: 0.8331 - val_specificity: 0.8725 - val_miss_rate: 0.2063 - val_fall_out: 0.1275 - val_mcc: 0.6365
Epoch 13/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.2933 - accuracy: 0.8643 - recall: 0.7639 - precision: 0.7252 - AUROC: 0.9278 - AUPRC: 0.7892 - f1_score: 0.7441 - balanced_accuracy: 0.8316 - specificity: 0.8993 - miss_rate: 0.2361 - fall_out: 0.1007 - mcc: 0.6523 - val_loss: 0.3113 - val_accuracy: 0.8531 - val_recall: 0.7939 - val_precision: 0.6863 - val_AUROC: 0.9186 - val_AUPRC: 0.7643 - val_f1_score: 0.7362 - val_balanced_accuracy: 0.8338 - val_specificity: 0.8738 - val_miss_rate: 0.2061 - val_fall_out: 0.1262 - val_mcc: 0.6384
Epoch 14/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.2887 - accuracy: 0.8676 - recall: 0.7653 - precision: 0.7335 - AUROC: 0.9303 - AUPRC: 0.7996 - f1_score: 0.7491 - balanced_accuracy: 0.8343 - specificity: 0.9032 - miss_rate: 0.2347 - fall_out: 0.0968 - mcc: 0.6595 - val_loss: 0.3100 - val_accuracy: 0.8560 - val_recall: 0.7371 - val_precision: 0.7142 - val_AUROC: 0.9189 - val_AUPRC: 0.7672 - val_f1_score: 0.7255 - val_balanced_accuracy: 0.8172 - val_specificity: 0.8974 - val_miss_rate: 0.2629 - val_fall_out: 0.1026 - val_mcc: 0.6280
Epoch 15/1000
313/313 [==============================] - 13s 39ms/step - loss: 0.2858 - accuracy: 0.8687 - recall: 0.7660 - precision: 0.7360 - AUROC: 0.9318 - AUPRC: 0.8022 - f1_score: 0.7507 - balanced_accuracy: 0.8352 - specificity: 0.9044 - miss_rate: 0.2340 - fall_out: 0.0956 - mcc: 0.6618 - val_loss: 0.3095 - val_accuracy: 0.8559 - val_recall: 0.7533 - val_precision: 0.7075 - val_AUROC: 0.9191 - val_AUPRC: 0.7684 - val_f1_score: 0.7297 - val_balanced_accuracy: 0.8225 - val_specificity: 0.8916 - val_miss_rate: 0.2467 - val_fall_out: 0.1084 - val_mcc: 0.6322
Epoch 16/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.2839 - accuracy: 0.8687 - recall: 0.7635 - precision: 0.7373 - AUROC: 0.9327 - AUPRC: 0.8082 - f1_score: 0.7502 - balanced_accuracy: 0.8344 - specificity: 0.9053 - miss_rate: 0.2365 - fall_out: 0.0947 - mcc: 0.6614 - val_loss: 0.3090 - val_accuracy: 0.8551 - val_recall: 0.7603 - val_precision: 0.7027 - val_AUROC: 0.9187 - val_AUPRC: 0.7645 - val_f1_score: 0.7304 - val_balanced_accuracy: 0.8242 - val_specificity: 0.8881 - val_miss_rate: 0.2397 - val_fall_out: 0.1119 - val_mcc: 0.6324
Epoch 17/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.2766 - accuracy: 0.8735 - recall: 0.7697 - precision: 0.7477 - AUROC: 0.9365 - AUPRC: 0.8191 - f1_score: 0.7585 - balanced_accuracy: 0.8397 - specificity: 0.9096 - miss_rate: 0.2303 - fall_out: 0.0904 - mcc: 0.6730 - val_loss: 0.3081 - val_accuracy: 0.8540 - val_recall: 0.7303 - val_precision: 0.7116 - val_AUROC: 0.9195 - val_AUPRC: 0.7668 - val_f1_score: 0.7208 - val_balanced_accuracy: 0.8137 - val_specificity: 0.8970 - val_miss_rate: 0.2697 - val_fall_out: 0.1030 - val_mcc: 0.6221
Epoch 18/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.2755 - accuracy: 0.8741 - recall: 0.7738 - precision: 0.7474 - AUROC: 0.9370 - AUPRC: 0.8184 - f1_score: 0.7604 - balanced_accuracy: 0.8414 - specificity: 0.9090 - miss_rate: 0.2262 - fall_out: 0.0910 - mcc: 0.6752 - val_loss: 0.3122 - val_accuracy: 0.8542 - val_recall: 0.7225 - val_precision: 0.7156 - val_AUROC: 0.9176 - val_AUPRC: 0.7609 - val_f1_score: 0.7190 - val_balanced_accuracy: 0.8113 - val_specificity: 0.9001 - val_miss_rate: 0.2775 - val_fall_out: 0.0999 - val_mcc: 0.6206
Epoch 19/1000
313/313 [==============================] - 14s 43ms/step - loss: 0.2714 - accuracy: 0.8769 - recall: 0.7746 - precision: 0.7547 - AUROC: 0.9391 - AUPRC: 0.8264 - f1_score: 0.7646 - balanced_accuracy: 0.8435 - specificity: 0.9124 - miss_rate: 0.2254 - fall_out: 0.0876 - mcc: 0.6813 - val_loss: 0.3090 - val_accuracy: 0.8548 - val_recall: 0.7132 - val_precision: 0.7212 - val_AUROC: 0.9196 - val_AUPRC: 0.7678 - val_f1_score: 0.7172 - val_balanced_accuracy: 0.8086 - val_specificity: 0.9040 - val_miss_rate: 0.2868 - val_fall_out: 0.0960 - val_mcc: 0.6195
313/313 [==============================] - 8s 24ms/step - loss: 0.2456 - accuracy: 0.8959 - recall: 0.7903 - precision: 0.8031 - AUROC: 0.9553 - AUPRC: 0.8773 - f1_score: 0.7967 - balanced_accuracy: 0.8614 - specificity: 0.9326 - miss_rate: 0.2097 - fall_out: 0.0674 - mcc: 0.7267
79/79 [==============================] - 4s 50ms/step - loss: 0.3090 - accuracy: 0.8548 - recall: 0.7132 - precision: 0.7212 - AUROC: 0.9196 - AUPRC: 0.7678 - f1_score: 0.7172 - balanced_accuracy: 0.8086 - specificity: 0.9040 - miss_rate: 0.2868 - fall_out: 0.0960 - mcc: 0.6195
-- HOLDOUT 7 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 19s 53ms/step - loss: 11.4783 - accuracy: 0.8029 - recall: 0.5663 - precision: 0.6318 - AUROC: 0.8593 - AUPRC: 0.6234 - f1_score: 0.5972 - balanced_accuracy: 0.7257 - specificity: 0.8852 - miss_rate: 0.4337 - fall_out: 0.1148 - mcc: 0.4685 - val_loss: 1.8034 - val_accuracy: 0.8248 - val_recall: 0.6413 - val_precision: 0.6673 - val_AUROC: 0.8899 - val_AUPRC: 0.6549 - val_f1_score: 0.6540 - val_balanced_accuracy: 0.7650 - val_specificity: 0.8887 - val_miss_rate: 0.3587 - val_fall_out: 0.1113 - val_mcc: 0.5370
Epoch 2/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.9346 - accuracy: 0.8323 - recall: 0.7176 - precision: 0.6616 - AUROC: 0.8912 - AUPRC: 0.6781 - f1_score: 0.6885 - balanced_accuracy: 0.7949 - specificity: 0.8723 - miss_rate: 0.2824 - fall_out: 0.1277 - mcc: 0.5749 - val_loss: 2.0877 - val_accuracy: 0.8389 - val_recall: 0.7873 - val_precision: 0.6567 - val_AUROC: 0.8988 - val_AUPRC: 0.6851 - val_f1_score: 0.7161 - val_balanced_accuracy: 0.8220 - val_specificity: 0.8568 - val_miss_rate: 0.2127 - val_fall_out: 0.1432 - val_mcc: 0.6097
Epoch 3/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.7458 - accuracy: 0.8366 - recall: 0.7479 - precision: 0.6625 - AUROC: 0.8963 - AUPRC: 0.6890 - f1_score: 0.7027 - balanced_accuracy: 0.8077 - specificity: 0.8675 - miss_rate: 0.2521 - fall_out: 0.1325 - mcc: 0.5926 - val_loss: 1.4690 - val_accuracy: 0.8396 - val_recall: 0.7848 - val_precision: 0.6589 - val_AUROC: 0.9020 - val_AUPRC: 0.7035 - val_f1_score: 0.7163 - val_balanced_accuracy: 0.8217 - val_specificity: 0.8586 - val_miss_rate: 0.2152 - val_fall_out: 0.1414 - val_mcc: 0.6102
Epoch 4/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.6226 - accuracy: 0.8396 - recall: 0.7539 - precision: 0.6677 - AUROC: 0.9010 - AUPRC: 0.7021 - f1_score: 0.7082 - balanced_accuracy: 0.8117 - specificity: 0.8694 - miss_rate: 0.2461 - fall_out: 0.1306 - mcc: 0.6003 - val_loss: 1.8196 - val_accuracy: 0.8426 - val_recall: 0.7904 - val_precision: 0.6640 - val_AUROC: 0.9049 - val_AUPRC: 0.7104 - val_f1_score: 0.7217 - val_balanced_accuracy: 0.8256 - val_specificity: 0.8608 - val_miss_rate: 0.2096 - val_fall_out: 0.1392 - val_mcc: 0.6176
Epoch 5/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.5375 - accuracy: 0.8410 - recall: 0.7580 - precision: 0.6697 - AUROC: 0.9018 - AUPRC: 0.7005 - f1_score: 0.7111 - balanced_accuracy: 0.8139 - specificity: 0.8699 - miss_rate: 0.2420 - fall_out: 0.1301 - mcc: 0.6042 - val_loss: 1.3225 - val_accuracy: 0.8442 - val_recall: 0.7322 - val_precision: 0.6857 - val_AUROC: 0.9057 - val_AUPRC: 0.7172 - val_f1_score: 0.7082 - val_balanced_accuracy: 0.8077 - val_specificity: 0.8832 - val_miss_rate: 0.2678 - val_fall_out: 0.1168 - val_mcc: 0.6027
Epoch 6/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.4676 - accuracy: 0.8436 - recall: 0.7542 - precision: 0.6768 - AUROC: 0.9046 - AUPRC: 0.7093 - f1_score: 0.7134 - balanced_accuracy: 0.8144 - specificity: 0.8747 - miss_rate: 0.2458 - fall_out: 0.1253 - mcc: 0.6079 - val_loss: 1.6265 - val_accuracy: 0.8447 - val_recall: 0.7741 - val_precision: 0.6732 - val_AUROC: 0.9063 - val_AUPRC: 0.7123 - val_f1_score: 0.7201 - val_balanced_accuracy: 0.8217 - val_specificity: 0.8692 - val_miss_rate: 0.2259 - val_fall_out: 0.1308 - val_mcc: 0.6162
Epoch 7/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.4119 - accuracy: 0.8464 - recall: 0.7529 - precision: 0.6838 - AUROC: 0.9063 - AUPRC: 0.7162 - f1_score: 0.7167 - balanced_accuracy: 0.8159 - specificity: 0.8789 - miss_rate: 0.2471 - fall_out: 0.1211 - mcc: 0.6130 - val_loss: 1.2333 - val_accuracy: 0.8432 - val_recall: 0.7857 - val_precision: 0.6666 - val_AUROC: 0.9067 - val_AUPRC: 0.7154 - val_f1_score: 0.7213 - val_balanced_accuracy: 0.8245 - val_specificity: 0.8632 - val_miss_rate: 0.2143 - val_fall_out: 0.1368 - val_mcc: 0.6172
Epoch 8/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.3616 - accuracy: 0.8478 - recall: 0.7616 - precision: 0.6844 - AUROC: 0.9072 - AUPRC: 0.7159 - f1_score: 0.7209 - balanced_accuracy: 0.8197 - specificity: 0.8778 - miss_rate: 0.2384 - fall_out: 0.1222 - mcc: 0.6184 - val_loss: 1.5777 - val_accuracy: 0.8457 - val_recall: 0.7749 - val_precision: 0.6752 - val_AUROC: 0.9079 - val_AUPRC: 0.7210 - val_f1_score: 0.7216 - val_balanced_accuracy: 0.8226 - val_specificity: 0.8703 - val_miss_rate: 0.2251 - val_fall_out: 0.1297 - val_mcc: 0.6184
Epoch 9/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.3188 - accuracy: 0.8489 - recall: 0.7636 - precision: 0.6862 - AUROC: 0.9097 - AUPRC: 0.7235 - f1_score: 0.7228 - balanced_accuracy: 0.8211 - specificity: 0.8785 - miss_rate: 0.2364 - fall_out: 0.1215 - mcc: 0.6210 - val_loss: 1.1270 - val_accuracy: 0.8464 - val_recall: 0.7708 - val_precision: 0.6781 - val_AUROC: 0.9091 - val_AUPRC: 0.7248 - val_f1_score: 0.7215 - val_balanced_accuracy: 0.8217 - val_specificity: 0.8727 - val_miss_rate: 0.2292 - val_fall_out: 0.1273 - val_mcc: 0.6184
Epoch 10/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.2818 - accuracy: 0.8499 - recall: 0.7605 - precision: 0.6899 - AUROC: 0.9108 - AUPRC: 0.7269 - f1_score: 0.7235 - balanced_accuracy: 0.8208 - specificity: 0.8810 - miss_rate: 0.2395 - fall_out: 0.1190 - mcc: 0.6222 - val_loss: 1.4002 - val_accuracy: 0.8445 - val_recall: 0.7002 - val_precision: 0.6983 - val_AUROC: 0.9077 - val_AUPRC: 0.7176 - val_f1_score: 0.6993 - val_balanced_accuracy: 0.7975 - val_specificity: 0.8947 - val_miss_rate: 0.2998 - val_fall_out: 0.1053 - val_mcc: 0.5944
Epoch 11/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.2471 - accuracy: 0.8511 - recall: 0.7684 - precision: 0.6901 - AUROC: 0.9122 - AUPRC: 0.7293 - f1_score: 0.7271 - balanced_accuracy: 0.8242 - specificity: 0.8799 - miss_rate: 0.2316 - fall_out: 0.1201 - mcc: 0.6269 - val_loss: 1.1153 - val_accuracy: 0.8461 - val_recall: 0.7780 - val_precision: 0.6753 - val_AUROC: 0.9096 - val_AUPRC: 0.7269 - val_f1_score: 0.7230 - val_balanced_accuracy: 0.8239 - val_specificity: 0.8698 - val_miss_rate: 0.2220 - val_fall_out: 0.1302 - val_mcc: 0.6202
Epoch 12/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.2144 - accuracy: 0.8523 - recall: 0.7658 - precision: 0.6938 - AUROC: 0.9132 - AUPRC: 0.7307 - f1_score: 0.7280 - balanced_accuracy: 0.8241 - specificity: 0.8824 - miss_rate: 0.2342 - fall_out: 0.1176 - mcc: 0.6285 - val_loss: 1.2600 - val_accuracy: 0.8472 - val_recall: 0.7679 - val_precision: 0.6810 - val_AUROC: 0.9098 - val_AUPRC: 0.7279 - val_f1_score: 0.7218 - val_balanced_accuracy: 0.8214 - val_specificity: 0.8748 - val_miss_rate: 0.2321 - val_fall_out: 0.1252 - val_mcc: 0.6192
Epoch 13/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.1850 - accuracy: 0.8526 - recall: 0.7672 - precision: 0.6940 - AUROC: 0.9142 - AUPRC: 0.7370 - f1_score: 0.7288 - balanced_accuracy: 0.8248 - specificity: 0.8823 - miss_rate: 0.2328 - fall_out: 0.1177 - mcc: 0.6294 - val_loss: 1.1113 - val_accuracy: 0.8473 - val_recall: 0.7832 - val_precision: 0.6763 - val_AUROC: 0.9101 - val_AUPRC: 0.7287 - val_f1_score: 0.7259 - val_balanced_accuracy: 0.8264 - val_specificity: 0.8696 - val_miss_rate: 0.2168 - val_fall_out: 0.1304 - val_mcc: 0.6240
Epoch 14/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.1572 - accuracy: 0.8532 - recall: 0.7714 - precision: 0.6941 - AUROC: 0.9150 - AUPRC: 0.7359 - f1_score: 0.7307 - balanced_accuracy: 0.8265 - specificity: 0.8817 - miss_rate: 0.2286 - fall_out: 0.1183 - mcc: 0.6319 - val_loss: 1.2257 - val_accuracy: 0.8474 - val_recall: 0.7658 - val_precision: 0.6820 - val_AUROC: 0.9108 - val_AUPRC: 0.7296 - val_f1_score: 0.7215 - val_balanced_accuracy: 0.8208 - val_specificity: 0.8758 - val_miss_rate: 0.2342 - val_fall_out: 0.1242 - val_mcc: 0.6188
Epoch 15/1000
313/313 [==============================] - 17s 51ms/step - loss: 1.1291 - accuracy: 0.8544 - recall: 0.7753 - precision: 0.6956 - AUROC: 0.9162 - AUPRC: 0.7397 - f1_score: 0.7333 - balanced_accuracy: 0.8286 - specificity: 0.8819 - miss_rate: 0.2247 - fall_out: 0.1181 - mcc: 0.6353 - val_loss: 1.1274 - val_accuracy: 0.8477 - val_recall: 0.7793 - val_precision: 0.6784 - val_AUROC: 0.9110 - val_AUPRC: 0.7317 - val_f1_score: 0.7254 - val_balanced_accuracy: 0.8254 - val_specificity: 0.8715 - val_miss_rate: 0.2207 - val_fall_out: 0.1285 - val_mcc: 0.6235
313/313 [==============================] - 8s 26ms/step - loss: 1.1051 - accuracy: 0.8637 - recall: 0.8064 - precision: 0.7067 - AUROC: 0.9260 - AUPRC: 0.7654 - f1_score: 0.7533 - balanced_accuracy: 0.8450 - specificity: 0.8836 - miss_rate: 0.1936 - fall_out: 0.1164 - mcc: 0.6624
79/79 [==============================] - 4s 54ms/step - loss: 1.1274 - accuracy: 0.8477 - recall: 0.7793 - precision: 0.6784 - AUROC: 0.9110 - AUPRC: 0.7317 - f1_score: 0.7254 - balanced_accuracy: 0.8254 - specificity: 0.8715 - miss_rate: 0.2207 - fall_out: 0.1285 - mcc: 0.6235
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_6\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_6\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 136, 'dropout_2': 0.5, 'units_3': 136, 'dropout_3': 0.5, 'units_4': 136, 'dropout_4': 0.5}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_6\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_6\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 4, 'filters_0': 160, 'kernel_size_0': 16, 'conv_dropout_0': 0.2, 'dense_depth': 4, 'units_0': 128, 'dense_dropout_0': 0.4, 'units_1': 128, 'dense_dropout_1': 0.3, 'units_2': 32, 'dense_dropout_2': 0.4, 'filters_1': 32, 'kernel_size_1': 6, 'conv_dropout_1': 0.0, 'units_3': 16, 'dense_dropout_3': 0.3, 'filters_2': 72, 'kernel_size_2': 6, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.0, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.1, 'filters_5': 32, 'kernel_size_5': 6, 'conv_dropout_5': 0.0, 'units_4': 16, 'dense_dropout_4': 0.4}
- Removing uncorrelated/correlated features: ['ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 1) 137
=================================================================
Total params: 76,569
Trainable params: 76,569
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 10400
dropout_4 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 32) 30752
dropout_5 (Dropout) (None, 128, 32) 0
max_pooling1d_2 (MaxPooling (None, 64, 32) 0
1D)
conv1d_3 (Conv1D) (None, 64, 72) 13896
dropout_6 (Dropout) (None, 64, 72) 0
max_pooling1d_3 (MaxPooling (None, 32, 72) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 3464
dropout_7 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
flatten_1 (Flatten) (None, 128) 0
dense_6 (Dense) (None, 128) 16512
dropout_8 (Dropout) (None, 128) 0
dense_7 (Dense) (None, 128) 16512
dropout_9 (Dropout) (None, 128) 0
dense_8 (Dense) (None, 32) 4128
dropout_10 (Dropout) (None, 32) 0
dense_9 (Dense) (None, 16) 528
dropout_11 (Dropout) (None, 16) 0
dense_10 (Dense) (None, 1) 17
=================================================================
Total params: 96,209
Trainable params: 96,209
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 10400 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 32) 30752 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 72) 13896 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 72) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 72) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 3464 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_7[0][0]']
flatten_1 (Flatten) (None, 128) 0 ['max_pooling1d_4[0][0]']
dense_6 (Dense) (None, 128) 16512 ['flatten_1[0][0]']
dropout_8 (Dropout) (None, 128) 0 ['dense_6[0][0]']
dense_7 (Dense) (None, 128) 16512 ['dropout_8[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_7[0][0]']
dense_8 (Dense) (None, 32) 4128 ['dropout_9[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
dropout_10 (Dropout) (None, 32) 0 ['dense_8[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
dense_9 (Dense) (None, 16) 528 ['dropout_10[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_11 (Dropout) (None, 16) 0 ['dense_9[0][0]']
concatenate (Concatenate) (None, 152) 0 ['dropout_3[0][0]',
'dropout_11[0][0]']
dense_11 (Dense) (None, 64) 9792 ['concatenate[0][0]']
dense_12 (Dense) (None, 1) 65 ['dense_11[0][0]']
==================================================================================================
Total params: 182,481
Trainable params: 182,481
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 17s 44ms/step - loss: 0.3773 - accuracy: 0.8243 - recall: 0.6545 - precision: 0.6614 - AUROC: 0.8803 - AUPRC: 0.6620 - f1_score: 0.6579 - balanced_accuracy: 0.7690 - specificity: 0.8834 - miss_rate: 0.3455 - fall_out: 0.1166 - mcc: 0.5398 - val_loss: 0.3438 - val_accuracy: 0.8353 - val_recall: 0.6868 - val_precision: 0.6789 - val_AUROC: 0.8980 - val_AUPRC: 0.6897 - val_f1_score: 0.6829 - val_balanced_accuracy: 0.7869 - val_specificity: 0.8870 - val_miss_rate: 0.3132 - val_fall_out: 0.1130 - val_mcc: 0.5717
Epoch 2/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.3393 - accuracy: 0.8391 - recall: 0.7168 - precision: 0.6783 - AUROC: 0.9011 - AUPRC: 0.7049 - f1_score: 0.6970 - balanced_accuracy: 0.7992 - specificity: 0.8817 - miss_rate: 0.2832 - fall_out: 0.1183 - mcc: 0.5880 - val_loss: 0.3324 - val_accuracy: 0.8432 - val_recall: 0.7679 - val_precision: 0.6716 - val_AUROC: 0.9052 - val_AUPRC: 0.7138 - val_f1_score: 0.7165 - val_balanced_accuracy: 0.8186 - val_specificity: 0.8694 - val_miss_rate: 0.2321 - val_fall_out: 0.1306 - val_mcc: 0.6114
Epoch 3/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.3294 - accuracy: 0.8446 - recall: 0.7313 - precision: 0.6870 - AUROC: 0.9066 - AUPRC: 0.7193 - f1_score: 0.7085 - balanced_accuracy: 0.8077 - specificity: 0.8841 - miss_rate: 0.2687 - fall_out: 0.1159 - mcc: 0.6033 - val_loss: 0.3326 - val_accuracy: 0.8431 - val_recall: 0.7392 - val_precision: 0.6805 - val_AUROC: 0.9054 - val_AUPRC: 0.7195 - val_f1_score: 0.7086 - val_balanced_accuracy: 0.8092 - val_specificity: 0.8792 - val_miss_rate: 0.2608 - val_fall_out: 0.1208 - val_mcc: 0.6025
Epoch 4/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3232 - accuracy: 0.8485 - recall: 0.7453 - precision: 0.6917 - AUROC: 0.9103 - AUPRC: 0.7304 - f1_score: 0.7175 - balanced_accuracy: 0.8149 - specificity: 0.8844 - miss_rate: 0.2547 - fall_out: 0.1156 - mcc: 0.6150 - val_loss: 0.3295 - val_accuracy: 0.8450 - val_recall: 0.7890 - val_precision: 0.6695 - val_AUROC: 0.9077 - val_AUPRC: 0.7266 - val_f1_score: 0.7243 - val_balanced_accuracy: 0.8267 - val_specificity: 0.8644 - val_miss_rate: 0.2110 - val_fall_out: 0.1356 - val_mcc: 0.6215
Epoch 5/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3166 - accuracy: 0.8510 - recall: 0.7499 - precision: 0.6962 - AUROC: 0.9142 - AUPRC: 0.7431 - f1_score: 0.7220 - balanced_accuracy: 0.8180 - specificity: 0.8861 - miss_rate: 0.2501 - fall_out: 0.1139 - mcc: 0.6212 - val_loss: 0.3253 - val_accuracy: 0.8450 - val_recall: 0.7436 - val_precision: 0.6836 - val_AUROC: 0.9092 - val_AUPRC: 0.7272 - val_f1_score: 0.7124 - val_balanced_accuracy: 0.8119 - val_specificity: 0.8802 - val_miss_rate: 0.2564 - val_fall_out: 0.1198 - val_mcc: 0.6075
Epoch 6/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.3123 - accuracy: 0.8534 - recall: 0.7578 - precision: 0.6993 - AUROC: 0.9165 - AUPRC: 0.7483 - f1_score: 0.7274 - balanced_accuracy: 0.8222 - specificity: 0.8866 - miss_rate: 0.2422 - fall_out: 0.1134 - mcc: 0.6282 - val_loss: 0.3287 - val_accuracy: 0.8421 - val_recall: 0.6574 - val_precision: 0.7097 - val_AUROC: 0.9094 - val_AUPRC: 0.7298 - val_f1_score: 0.6825 - val_balanced_accuracy: 0.7819 - val_specificity: 0.9064 - val_miss_rate: 0.3426 - val_fall_out: 0.0936 - val_mcc: 0.5784
Epoch 7/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.3078 - accuracy: 0.8560 - recall: 0.7564 - precision: 0.7064 - AUROC: 0.9192 - AUPRC: 0.7563 - f1_score: 0.7305 - balanced_accuracy: 0.8235 - specificity: 0.8906 - miss_rate: 0.2436 - fall_out: 0.1094 - mcc: 0.6331 - val_loss: 0.3221 - val_accuracy: 0.8475 - val_recall: 0.7751 - val_precision: 0.6793 - val_AUROC: 0.9117 - val_AUPRC: 0.7335 - val_f1_score: 0.7240 - val_balanced_accuracy: 0.8239 - val_specificity: 0.8727 - val_miss_rate: 0.2249 - val_fall_out: 0.1273 - val_mcc: 0.6218
Epoch 8/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.3035 - accuracy: 0.8584 - recall: 0.7610 - precision: 0.7109 - AUROC: 0.9217 - AUPRC: 0.7657 - f1_score: 0.7351 - balanced_accuracy: 0.8266 - specificity: 0.8923 - miss_rate: 0.2390 - fall_out: 0.1077 - mcc: 0.6393 - val_loss: 0.3237 - val_accuracy: 0.8474 - val_recall: 0.7194 - val_precision: 0.6986 - val_AUROC: 0.9107 - val_AUPRC: 0.7325 - val_f1_score: 0.7088 - val_balanced_accuracy: 0.8057 - val_specificity: 0.8920 - val_miss_rate: 0.2806 - val_fall_out: 0.1080 - val_mcc: 0.6056
Epoch 9/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.2997 - accuracy: 0.8608 - recall: 0.7658 - precision: 0.7152 - AUROC: 0.9239 - AUPRC: 0.7708 - f1_score: 0.7396 - balanced_accuracy: 0.8298 - specificity: 0.8939 - miss_rate: 0.2342 - fall_out: 0.1061 - mcc: 0.6455 - val_loss: 0.3199 - val_accuracy: 0.8501 - val_recall: 0.7941 - val_precision: 0.6793 - val_AUROC: 0.9136 - val_AUPRC: 0.7396 - val_f1_score: 0.7322 - val_balanced_accuracy: 0.8318 - val_specificity: 0.8696 - val_miss_rate: 0.2059 - val_fall_out: 0.1304 - val_mcc: 0.6327
Epoch 10/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.2940 - accuracy: 0.8635 - recall: 0.7716 - precision: 0.7197 - AUROC: 0.9269 - AUPRC: 0.7819 - f1_score: 0.7447 - balanced_accuracy: 0.8335 - specificity: 0.8954 - miss_rate: 0.2284 - fall_out: 0.1046 - mcc: 0.6524 - val_loss: 0.3243 - val_accuracy: 0.8502 - val_recall: 0.7927 - val_precision: 0.6800 - val_AUROC: 0.9132 - val_AUPRC: 0.7426 - val_f1_score: 0.7320 - val_balanced_accuracy: 0.8314 - val_specificity: 0.8702 - val_miss_rate: 0.2073 - val_fall_out: 0.1298 - val_mcc: 0.6325
Epoch 11/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2896 - accuracy: 0.8638 - recall: 0.7712 - precision: 0.7206 - AUROC: 0.9291 - AUPRC: 0.7885 - f1_score: 0.7451 - balanced_accuracy: 0.8336 - specificity: 0.8959 - miss_rate: 0.2288 - fall_out: 0.1041 - mcc: 0.6529 - val_loss: 0.3219 - val_accuracy: 0.8480 - val_recall: 0.7727 - val_precision: 0.6812 - val_AUROC: 0.9124 - val_AUPRC: 0.7415 - val_f1_score: 0.7241 - val_balanced_accuracy: 0.8234 - val_specificity: 0.8742 - val_miss_rate: 0.2273 - val_fall_out: 0.1258 - val_mcc: 0.6221
313/313 [==============================] - 8s 24ms/step - loss: 0.2714 - accuracy: 0.8753 - recall: 0.8276 - precision: 0.7270 - AUROC: 0.9408 - AUPRC: 0.8238 - f1_score: 0.7741 - balanced_accuracy: 0.8598 - specificity: 0.8919 - miss_rate: 0.1724 - fall_out: 0.1081 - mcc: 0.6912
79/79 [==============================] - 4s 53ms/step - loss: 0.3219 - accuracy: 0.8480 - recall: 0.7727 - precision: 0.6812 - AUROC: 0.9124 - AUPRC: 0.7415 - f1_score: 0.7241 - balanced_accuracy: 0.8234 - specificity: 0.8742 - miss_rate: 0.2273 - fall_out: 0.1258 - mcc: 0.6221
-- HOLDOUT 8 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 20s 54ms/step - loss: 11.8332 - accuracy: 0.8036 - recall: 0.5568 - precision: 0.6367 - AUROC: 0.8594 - AUPRC: 0.6263 - f1_score: 0.5941 - balanced_accuracy: 0.7231 - specificity: 0.8894 - miss_rate: 0.4432 - fall_out: 0.1106 - mcc: 0.4671 - val_loss: 2.2502 - val_accuracy: 0.7839 - val_recall: 0.2730 - val_precision: 0.7122 - val_AUROC: 0.8949 - val_AUPRC: 0.6708 - val_f1_score: 0.3947 - val_balanced_accuracy: 0.6173 - val_specificity: 0.9616 - val_miss_rate: 0.7270 - val_fall_out: 0.0384 - val_mcc: 0.3439
Epoch 2/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.9351 - accuracy: 0.8306 - recall: 0.7198 - precision: 0.6568 - AUROC: 0.8904 - AUPRC: 0.6768 - f1_score: 0.6868 - balanced_accuracy: 0.7945 - specificity: 0.8691 - miss_rate: 0.2802 - fall_out: 0.1309 - mcc: 0.5722 - val_loss: 1.5684 - val_accuracy: 0.8427 - val_recall: 0.7596 - val_precision: 0.6731 - val_AUROC: 0.9020 - val_AUPRC: 0.6962 - val_f1_score: 0.7137 - val_balanced_accuracy: 0.8156 - val_specificity: 0.8717 - val_miss_rate: 0.2404 - val_fall_out: 0.1283 - val_mcc: 0.6080
Epoch 3/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.7423 - accuracy: 0.8361 - recall: 0.7517 - precision: 0.6603 - AUROC: 0.8968 - AUPRC: 0.6906 - f1_score: 0.7031 - balanced_accuracy: 0.8086 - specificity: 0.8655 - miss_rate: 0.2483 - fall_out: 0.1345 - mcc: 0.5929 - val_loss: 1.9425 - val_accuracy: 0.8428 - val_recall: 0.7691 - val_precision: 0.6705 - val_AUROC: 0.9048 - val_AUPRC: 0.7090 - val_f1_score: 0.7164 - val_balanced_accuracy: 0.8188 - val_specificity: 0.8685 - val_miss_rate: 0.2309 - val_fall_out: 0.1315 - val_mcc: 0.6111
Epoch 4/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.6213 - accuracy: 0.8379 - recall: 0.7589 - precision: 0.6623 - AUROC: 0.8997 - AUPRC: 0.6981 - f1_score: 0.7073 - balanced_accuracy: 0.8121 - specificity: 0.8653 - miss_rate: 0.2411 - fall_out: 0.1347 - mcc: 0.5985 - val_loss: 1.2971 - val_accuracy: 0.8442 - val_recall: 0.7739 - val_precision: 0.6722 - val_AUROC: 0.9066 - val_AUPRC: 0.7147 - val_f1_score: 0.7195 - val_balanced_accuracy: 0.8213 - val_specificity: 0.8687 - val_miss_rate: 0.2261 - val_fall_out: 0.1313 - val_mcc: 0.6153
Epoch 5/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.5302 - accuracy: 0.8413 - recall: 0.7584 - precision: 0.6701 - AUROC: 0.9026 - AUPRC: 0.7061 - f1_score: 0.7115 - balanced_accuracy: 0.8143 - specificity: 0.8701 - miss_rate: 0.2416 - fall_out: 0.1299 - mcc: 0.6048 - val_loss: 1.7158 - val_accuracy: 0.8453 - val_recall: 0.7813 - val_precision: 0.6724 - val_AUROC: 0.9077 - val_AUPRC: 0.7178 - val_f1_score: 0.7228 - val_balanced_accuracy: 0.8244 - val_specificity: 0.8675 - val_miss_rate: 0.2187 - val_fall_out: 0.1325 - val_mcc: 0.6196
Epoch 6/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.4616 - accuracy: 0.8435 - recall: 0.7617 - precision: 0.6743 - AUROC: 0.9049 - AUPRC: 0.7112 - f1_score: 0.7154 - balanced_accuracy: 0.8169 - specificity: 0.8720 - miss_rate: 0.2383 - fall_out: 0.1280 - mcc: 0.6102 - val_loss: 1.1809 - val_accuracy: 0.8456 - val_recall: 0.7999 - val_precision: 0.6678 - val_AUROC: 0.9082 - val_AUPRC: 0.7204 - val_f1_score: 0.7279 - val_balanced_accuracy: 0.8307 - val_specificity: 0.8615 - val_miss_rate: 0.2001 - val_fall_out: 0.1385 - val_mcc: 0.6263
Epoch 7/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.4042 - accuracy: 0.8441 - recall: 0.7649 - precision: 0.6746 - AUROC: 0.9060 - AUPRC: 0.7153 - f1_score: 0.7169 - balanced_accuracy: 0.8183 - specificity: 0.8716 - miss_rate: 0.2351 - fall_out: 0.1284 - mcc: 0.6122 - val_loss: 1.6249 - val_accuracy: 0.8466 - val_recall: 0.7640 - val_precision: 0.6808 - val_AUROC: 0.9088 - val_AUPRC: 0.7219 - val_f1_score: 0.7200 - val_balanced_accuracy: 0.8197 - val_specificity: 0.8754 - val_miss_rate: 0.2360 - val_fall_out: 0.1246 - val_mcc: 0.6168
Epoch 8/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.3595 - accuracy: 0.8456 - recall: 0.7635 - precision: 0.6785 - AUROC: 0.9073 - AUPRC: 0.7183 - f1_score: 0.7185 - balanced_accuracy: 0.8188 - specificity: 0.8741 - miss_rate: 0.2365 - fall_out: 0.1259 - mcc: 0.6146 - val_loss: 1.1362 - val_accuracy: 0.8476 - val_recall: 0.7599 - val_precision: 0.6844 - val_AUROC: 0.9092 - val_AUPRC: 0.7250 - val_f1_score: 0.7202 - val_balanced_accuracy: 0.8190 - val_specificity: 0.8781 - val_miss_rate: 0.2401 - val_fall_out: 0.1219 - val_mcc: 0.6174
Epoch 9/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.3162 - accuracy: 0.8475 - recall: 0.7630 - precision: 0.6832 - AUROC: 0.9092 - AUPRC: 0.7234 - f1_score: 0.7209 - balanced_accuracy: 0.8199 - specificity: 0.8769 - miss_rate: 0.2370 - fall_out: 0.1231 - mcc: 0.6182 - val_loss: 1.4866 - val_accuracy: 0.8471 - val_recall: 0.7245 - val_precision: 0.6957 - val_AUROC: 0.9100 - val_AUPRC: 0.7265 - val_f1_score: 0.7098 - val_balanced_accuracy: 0.8071 - val_specificity: 0.8897 - val_miss_rate: 0.2755 - val_fall_out: 0.1103 - val_mcc: 0.6063
Epoch 10/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.2800 - accuracy: 0.8487 - recall: 0.7633 - precision: 0.6859 - AUROC: 0.9105 - AUPRC: 0.7268 - f1_score: 0.7225 - balanced_accuracy: 0.8208 - specificity: 0.8784 - miss_rate: 0.2367 - fall_out: 0.1216 - mcc: 0.6206 - val_loss: 1.1823 - val_accuracy: 0.8486 - val_recall: 0.7642 - val_precision: 0.6854 - val_AUROC: 0.9111 - val_AUPRC: 0.7295 - val_f1_score: 0.7227 - val_balanced_accuracy: 0.8211 - val_specificity: 0.8779 - val_miss_rate: 0.2358 - val_fall_out: 0.1221 - val_mcc: 0.6207
313/313 [==============================] - 9s 27ms/step - loss: 1.1711 - accuracy: 0.8560 - recall: 0.7829 - precision: 0.6967 - AUROC: 0.9197 - AUPRC: 0.7511 - f1_score: 0.7373 - balanced_accuracy: 0.8321 - specificity: 0.8814 - miss_rate: 0.2171 - fall_out: 0.1186 - mcc: 0.6406
79/79 [==============================] - 5s 56ms/step - loss: 1.1823 - accuracy: 0.8486 - recall: 0.7642 - precision: 0.6854 - AUROC: 0.9111 - AUPRC: 0.7295 - f1_score: 0.7227 - balanced_accuracy: 0.8211 - specificity: 0.8779 - miss_rate: 0.2358 - fall_out: 0.1221 - mcc: 0.6207
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_7\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_7\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 4, 'units_0': 72, 'dropout_0': 0.5, 'units_1': 104, 'dropout_1': 0.4, 'units_2': 40, 'dropout_2': 0.5, 'units_3': 40, 'dropout_3': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_7\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_7\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 8, 'conv_dropout_0': 0.2, 'dense_depth': 1, 'units_0': 16, 'dense_dropout_0': 0.3, 'filters_1': 160, 'kernel_size_1': 16, 'conv_dropout_1': 0.2, 'filters_2': 8, 'kernel_size_2': 16, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.2, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.2, 'units_1': 16, 'dense_dropout_1': 0.3, 'units_2': 64, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.3, 'filters_5': 96, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 72) 40464
dropout_3 (Dropout) (None, 72) 0
dense_5 (Dense) (None, 104) 7592
dropout_4 (Dropout) (None, 104) 0
dense_6 (Dense) (None, 40) 4200
dropout_5 (Dropout) (None, 40) 0
dense_7 (Dense) (None, 40) 1640
dropout_6 (Dropout) (None, 40) 0
dense_8 (Dense) (None, 1) 41
=================================================================
Total params: 53,937
Trainable params: 53,937
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 5280
dropout_7 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 160) 409760
dropout_8 (Dropout) (None, 128, 160) 0
max_pooling1d_2 (MaxPooling (None, 64, 160) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 20488
dropout_9 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 392
dropout_10 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 8) 392
dropout_11 (Dropout) (None, 16, 8) 0
max_pooling1d_5 (MaxPooling (None, 8, 8) 0
1D)
conv1d_6 (Conv1D) (None, 8, 96) 4704
dropout_12 (Dropout) (None, 8, 96) 0
max_pooling1d_6 (MaxPooling (None, 4, 96) 0
1D)
flatten_1 (Flatten) (None, 384) 0
dense_9 (Dense) (None, 16) 6160
dropout_13 (Dropout) (None, 16) 0
dense_10 (Dense) (None, 1) 17
=================================================================
Total params: 447,193
Trainable params: 447,193
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 5280 ['sequence_data[0][0]']
dropout_7 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_7[0][0]']
conv1d_2 (Conv1D) (None, 128, 160) 409760 ['max_pooling1d_1[0][0]']
dropout_8 (Dropout) (None, 128, 160) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 160) 0 ['dropout_8[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 20488 ['max_pooling1d_2[0][0]']
dropout_9 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_9[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 392 ['max_pooling1d_3[0][0]']
dropout_10 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_10[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
conv1d_5 (Conv1D) (None, 16, 8) 392 ['max_pooling1d_4[0][0]']
dense_4 (Dense) (None, 72) 40464 ['epigenomic_data[0][0]']
dropout_11 (Dropout) (None, 16, 8) 0 ['conv1d_5[0][0]']
dropout_3 (Dropout) (None, 72) 0 ['dense_4[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 8) 0 ['dropout_11[0][0]']
dense_5 (Dense) (None, 104) 7592 ['dropout_3[0][0]']
conv1d_6 (Conv1D) (None, 8, 96) 4704 ['max_pooling1d_5[0][0]']
dropout_4 (Dropout) (None, 104) 0 ['dense_5[0][0]']
dropout_12 (Dropout) (None, 8, 96) 0 ['conv1d_6[0][0]']
dense_6 (Dense) (None, 40) 4200 ['dropout_4[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 96) 0 ['dropout_12[0][0]']
dropout_5 (Dropout) (None, 40) 0 ['dense_6[0][0]']
flatten_1 (Flatten) (None, 384) 0 ['max_pooling1d_6[0][0]']
dense_7 (Dense) (None, 40) 1640 ['dropout_5[0][0]']
dense_9 (Dense) (None, 16) 6160 ['flatten_1[0][0]']
dropout_6 (Dropout) (None, 40) 0 ['dense_7[0][0]']
dropout_13 (Dropout) (None, 16) 0 ['dense_9[0][0]']
concatenate (Concatenate) (None, 56) 0 ['dropout_6[0][0]',
'dropout_13[0][0]']
dense_11 (Dense) (None, 64) 3648 ['concatenate[0][0]']
dense_12 (Dense) (None, 1) 65 ['dense_11[0][0]']
==================================================================================================
Total params: 504,785
Trainable params: 504,785
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 20s 54ms/step - loss: 0.4187 - accuracy: 0.7989 - recall: 0.5194 - precision: 0.6350 - AUROC: 0.8472 - AUPRC: 0.6111 - f1_score: 0.5714 - balanced_accuracy: 0.7077 - specificity: 0.8961 - miss_rate: 0.4806 - fall_out: 0.1039 - mcc: 0.4455 - val_loss: 0.3765 - val_accuracy: 0.8365 - val_recall: 0.7945 - val_precision: 0.6500 - val_AUROC: 0.8956 - val_AUPRC: 0.6788 - val_f1_score: 0.7150 - val_balanced_accuracy: 0.8228 - val_specificity: 0.8511 - val_miss_rate: 0.2055 - val_fall_out: 0.1489 - val_mcc: 0.6079
Epoch 2/1000
313/313 [==============================] - 17s 51ms/step - loss: 0.3598 - accuracy: 0.8271 - recall: 0.6926 - precision: 0.6566 - AUROC: 0.8873 - AUPRC: 0.6695 - f1_score: 0.6741 - balanced_accuracy: 0.7833 - specificity: 0.8740 - miss_rate: 0.3074 - fall_out: 0.1260 - mcc: 0.5569 - val_loss: 0.3455 - val_accuracy: 0.8408 - val_recall: 0.7745 - val_precision: 0.6643 - val_AUROC: 0.9003 - val_AUPRC: 0.6969 - val_f1_score: 0.7152 - val_balanced_accuracy: 0.8192 - val_specificity: 0.8638 - val_miss_rate: 0.2255 - val_fall_out: 0.1362 - val_mcc: 0.6090
Epoch 3/1000
313/313 [==============================] - 17s 52ms/step - loss: 0.3489 - accuracy: 0.8331 - recall: 0.7388 - precision: 0.6573 - AUROC: 0.8938 - AUPRC: 0.6845 - f1_score: 0.6957 - balanced_accuracy: 0.8024 - specificity: 0.8660 - miss_rate: 0.2612 - fall_out: 0.1340 - mcc: 0.5832 - val_loss: 0.3476 - val_accuracy: 0.8415 - val_recall: 0.7904 - val_precision: 0.6615 - val_AUROC: 0.9036 - val_AUPRC: 0.7073 - val_f1_score: 0.7202 - val_balanced_accuracy: 0.8248 - val_specificity: 0.8592 - val_miss_rate: 0.2096 - val_fall_out: 0.1408 - val_mcc: 0.6155
Epoch 4/1000
313/313 [==============================] - 17s 52ms/step - loss: 0.3438 - accuracy: 0.8376 - recall: 0.7488 - precision: 0.6646 - AUROC: 0.8969 - AUPRC: 0.6921 - f1_score: 0.7042 - balanced_accuracy: 0.8087 - specificity: 0.8685 - miss_rate: 0.2512 - fall_out: 0.1315 - mcc: 0.5948 - val_loss: 0.3555 - val_accuracy: 0.8427 - val_recall: 0.8007 - val_precision: 0.6614 - val_AUROC: 0.9051 - val_AUPRC: 0.7107 - val_f1_score: 0.7244 - val_balanced_accuracy: 0.8290 - val_specificity: 0.8574 - val_miss_rate: 0.1993 - val_fall_out: 0.1426 - val_mcc: 0.6212
313/313 [==============================] - 8s 25ms/step - loss: 0.3518 - accuracy: 0.8440 - recall: 0.8076 - precision: 0.6623 - AUROC: 0.9103 - AUPRC: 0.7291 - f1_score: 0.7277 - balanced_accuracy: 0.8322 - specificity: 0.8567 - miss_rate: 0.1924 - fall_out: 0.1433 - mcc: 0.6259
79/79 [==============================] - 5s 55ms/step - loss: 0.3555 - accuracy: 0.8427 - recall: 0.8007 - precision: 0.6614 - AUROC: 0.9051 - AUPRC: 0.7107 - f1_score: 0.7244 - balanced_accuracy: 0.8290 - specificity: 0.8574 - miss_rate: 0.1993 - fall_out: 0.1426 - mcc: 0.6212
-- HOLDOUT 9 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 20s 55ms/step - loss: 11.7879 - accuracy: 0.8020 - recall: 0.5622 - precision: 0.6305 - AUROC: 0.8583 - AUPRC: 0.6247 - f1_score: 0.5944 - balanced_accuracy: 0.7238 - specificity: 0.8854 - miss_rate: 0.4378 - fall_out: 0.1146 - mcc: 0.4653 - val_loss: 2.2823 - val_accuracy: 0.8330 - val_recall: 0.7404 - val_precision: 0.6566 - val_AUROC: 0.8962 - val_AUPRC: 0.6937 - val_f1_score: 0.6960 - val_balanced_accuracy: 0.8028 - val_specificity: 0.8652 - val_miss_rate: 0.2596 - val_fall_out: 0.1348 - val_mcc: 0.5834
Epoch 2/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.9302 - accuracy: 0.8317 - recall: 0.7227 - precision: 0.6586 - AUROC: 0.8913 - AUPRC: 0.6778 - f1_score: 0.6892 - balanced_accuracy: 0.7962 - specificity: 0.8697 - miss_rate: 0.2773 - fall_out: 0.1303 - mcc: 0.5753 - val_loss: 1.5829 - val_accuracy: 0.8339 - val_recall: 0.8270 - val_precision: 0.6373 - val_AUROC: 0.9012 - val_AUPRC: 0.7066 - val_f1_score: 0.7199 - val_balanced_accuracy: 0.8316 - val_specificity: 0.8362 - val_miss_rate: 0.1730 - val_fall_out: 0.1638 - val_mcc: 0.6150
Epoch 3/1000
313/313 [==============================] - 17s 52ms/step - loss: 1.7417 - accuracy: 0.8369 - recall: 0.7594 - precision: 0.6599 - AUROC: 0.8962 - AUPRC: 0.6871 - f1_score: 0.7062 - balanced_accuracy: 0.8116 - specificity: 0.8638 - miss_rate: 0.2406 - fall_out: 0.1362 - mcc: 0.5968 - val_loss: 1.8813 - val_accuracy: 0.8392 - val_recall: 0.8090 - val_precision: 0.6519 - val_AUROC: 0.9037 - val_AUPRC: 0.7136 - val_f1_score: 0.7220 - val_balanced_accuracy: 0.8293 - val_specificity: 0.8497 - val_miss_rate: 0.1910 - val_fall_out: 0.1503 - val_mcc: 0.6177
Epoch 4/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.6225 - accuracy: 0.8392 - recall: 0.7678 - precision: 0.6627 - AUROC: 0.9001 - AUPRC: 0.6996 - f1_score: 0.7114 - balanced_accuracy: 0.8159 - specificity: 0.8640 - miss_rate: 0.2322 - fall_out: 0.1360 - mcc: 0.6039 - val_loss: 1.3795 - val_accuracy: 0.8419 - val_recall: 0.7799 - val_precision: 0.6652 - val_AUROC: 0.9055 - val_AUPRC: 0.7183 - val_f1_score: 0.7180 - val_balanced_accuracy: 0.8217 - val_specificity: 0.8634 - val_miss_rate: 0.2201 - val_fall_out: 0.1366 - val_mcc: 0.6128
Epoch 5/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.5341 - accuracy: 0.8422 - recall: 0.7631 - precision: 0.6708 - AUROC: 0.9022 - AUPRC: 0.7042 - f1_score: 0.7140 - balanced_accuracy: 0.8164 - specificity: 0.8697 - miss_rate: 0.2369 - fall_out: 0.1303 - mcc: 0.6080 - val_loss: 1.7465 - val_accuracy: 0.8408 - val_recall: 0.8142 - val_precision: 0.6539 - val_AUROC: 0.9062 - val_AUPRC: 0.7196 - val_f1_score: 0.7253 - val_balanced_accuracy: 0.8322 - val_specificity: 0.8501 - val_miss_rate: 0.1858 - val_fall_out: 0.1499 - val_mcc: 0.6225
Epoch 6/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.4639 - accuracy: 0.8443 - recall: 0.7647 - precision: 0.6751 - AUROC: 0.9043 - AUPRC: 0.7087 - f1_score: 0.7171 - balanced_accuracy: 0.8183 - specificity: 0.8720 - miss_rate: 0.2353 - fall_out: 0.1280 - mcc: 0.6125 - val_loss: 1.1767 - val_accuracy: 0.8471 - val_recall: 0.7318 - val_precision: 0.6931 - val_AUROC: 0.9075 - val_AUPRC: 0.7199 - val_f1_score: 0.7119 - val_balanced_accuracy: 0.8095 - val_specificity: 0.8872 - val_miss_rate: 0.2682 - val_fall_out: 0.1128 - val_mcc: 0.6084
Epoch 7/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.4059 - accuracy: 0.8447 - recall: 0.7582 - precision: 0.6781 - AUROC: 0.9059 - AUPRC: 0.7126 - f1_score: 0.7159 - balanced_accuracy: 0.8165 - specificity: 0.8747 - miss_rate: 0.2418 - fall_out: 0.1253 - mcc: 0.6113 - val_loss: 1.5795 - val_accuracy: 0.8469 - val_recall: 0.7466 - val_precision: 0.6874 - val_AUROC: 0.9088 - val_AUPRC: 0.7246 - val_f1_score: 0.7157 - val_balanced_accuracy: 0.8142 - val_specificity: 0.8818 - val_miss_rate: 0.2534 - val_fall_out: 0.1182 - val_mcc: 0.6122
Epoch 8/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.3578 - accuracy: 0.8456 - recall: 0.7608 - precision: 0.6795 - AUROC: 0.9075 - AUPRC: 0.7166 - f1_score: 0.7179 - balanced_accuracy: 0.8180 - specificity: 0.8751 - miss_rate: 0.2392 - fall_out: 0.1249 - mcc: 0.6139 - val_loss: 1.1779 - val_accuracy: 0.8484 - val_recall: 0.7367 - val_precision: 0.6946 - val_AUROC: 0.9097 - val_AUPRC: 0.7293 - val_f1_score: 0.7150 - val_balanced_accuracy: 0.8120 - val_specificity: 0.8873 - val_miss_rate: 0.2633 - val_fall_out: 0.1127 - val_mcc: 0.6124
313/313 [==============================] - 9s 27ms/step - loss: 1.1658 - accuracy: 0.8530 - recall: 0.7403 - precision: 0.7051 - AUROC: 0.9173 - AUPRC: 0.7413 - f1_score: 0.7223 - balanced_accuracy: 0.8163 - specificity: 0.8923 - miss_rate: 0.2597 - fall_out: 0.1077 - mcc: 0.6228
79/79 [==============================] - 5s 57ms/step - loss: 1.1779 - accuracy: 0.8484 - recall: 0.7367 - precision: 0.6946 - AUROC: 0.9097 - AUPRC: 0.7293 - f1_score: 0.7150 - balanced_accuracy: 0.8120 - specificity: 0.8873 - miss_rate: 0.2633 - fall_out: 0.1127 - mcc: 0.6124
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_8\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_8\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 72, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.5, 'units_3': 8, 'dropout_3': 0.3, 'units_4': 8, 'dropout_4': 0.5}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_8\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_8\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 4, 'filters_0': 160, 'kernel_size_0': 16, 'conv_dropout_0': 0.2, 'dense_depth': 3, 'units_0': 128, 'dense_dropout_0': 0.3, 'filters_1': 56, 'kernel_size_1': 8, 'conv_dropout_1': 0.0, 'filters_2': 56, 'kernel_size_2': 6, 'conv_dropout_2': 0.0, 'units_1': 32, 'dense_dropout_1': 0.4, 'units_2': 16, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.3, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.0, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.0, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0, 'units_4': 16, 'dense_dropout_4': 0.3}
- Removing uncorrelated/correlated features: ['ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 1) 137
=================================================================
Total params: 76,569
Trainable params: 76,569
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 10400
dropout_4 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 56) 71736
dropout_5 (Dropout) (None, 128, 56) 0
max_pooling1d_2 (MaxPooling (None, 64, 56) 0
1D)
conv1d_3 (Conv1D) (None, 64, 56) 18872
dropout_6 (Dropout) (None, 64, 56) 0
max_pooling1d_3 (MaxPooling (None, 32, 56) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 2696
dropout_7 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
flatten_1 (Flatten) (None, 128) 0
dense_6 (Dense) (None, 128) 16512
dropout_8 (Dropout) (None, 128) 0
dense_7 (Dense) (None, 32) 4128
dropout_9 (Dropout) (None, 32) 0
dense_8 (Dense) (None, 16) 528
dropout_10 (Dropout) (None, 16) 0
dense_9 (Dense) (None, 1) 17
=================================================================
Total params: 124,889
Trainable params: 124,889
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 10400 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 56) 71736 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 56) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 56) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 56) 18872 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 56) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 56) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 2696 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_7[0][0]']
flatten_1 (Flatten) (None, 128) 0 ['max_pooling1d_4[0][0]']
dense_6 (Dense) (None, 128) 16512 ['flatten_1[0][0]']
dropout_8 (Dropout) (None, 128) 0 ['dense_6[0][0]']
dense_7 (Dense) (None, 32) 4128 ['dropout_8[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
dropout_9 (Dropout) (None, 32) 0 ['dense_7[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
dense_8 (Dense) (None, 16) 528 ['dropout_9[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_10 (Dropout) (None, 16) 0 ['dense_8[0][0]']
concatenate (Concatenate) (None, 152) 0 ['dropout_3[0][0]',
'dropout_10[0][0]']
dense_10 (Dense) (None, 64) 9792 ['concatenate[0][0]']
dense_11 (Dense) (None, 1) 65 ['dense_10[0][0]']
==================================================================================================
Total params: 211,161
Trainable params: 211,161
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 16s 43ms/step - loss: 0.3905 - accuracy: 0.8159 - recall: 0.6488 - precision: 0.6419 - AUROC: 0.8722 - AUPRC: 0.6428 - f1_score: 0.6453 - balanced_accuracy: 0.7614 - specificity: 0.8740 - miss_rate: 0.3512 - fall_out: 0.1260 - mcc: 0.5210 - val_loss: 0.3429 - val_accuracy: 0.8370 - val_recall: 0.7830 - val_precision: 0.6539 - val_AUROC: 0.8979 - val_AUPRC: 0.6892 - val_f1_score: 0.7127 - val_balanced_accuracy: 0.8194 - val_specificity: 0.8558 - val_miss_rate: 0.2170 - val_fall_out: 0.1442 - val_mcc: 0.6049
Epoch 2/1000
313/313 [==============================] - 14s 41ms/step - loss: 0.3490 - accuracy: 0.8353 - recall: 0.7107 - precision: 0.6708 - AUROC: 0.8949 - AUPRC: 0.6888 - f1_score: 0.6902 - balanced_accuracy: 0.7947 - specificity: 0.8787 - miss_rate: 0.2893 - fall_out: 0.1213 - mcc: 0.5786 - val_loss: 0.3363 - val_accuracy: 0.8403 - val_recall: 0.7665 - val_precision: 0.6656 - val_AUROC: 0.9022 - val_AUPRC: 0.7008 - val_f1_score: 0.7125 - val_balanced_accuracy: 0.8163 - val_specificity: 0.8660 - val_miss_rate: 0.2335 - val_fall_out: 0.1340 - val_mcc: 0.6056
Epoch 3/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3386 - accuracy: 0.8405 - recall: 0.7291 - precision: 0.6776 - AUROC: 0.9010 - AUPRC: 0.7043 - f1_score: 0.7024 - balanced_accuracy: 0.8042 - specificity: 0.8793 - miss_rate: 0.2709 - fall_out: 0.1207 - mcc: 0.5945 - val_loss: 0.3334 - val_accuracy: 0.8411 - val_recall: 0.7485 - val_precision: 0.6728 - val_AUROC: 0.9047 - val_AUPRC: 0.7142 - val_f1_score: 0.7086 - val_balanced_accuracy: 0.8109 - val_specificity: 0.8733 - val_miss_rate: 0.2515 - val_fall_out: 0.1267 - val_mcc: 0.6015
Epoch 4/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3332 - accuracy: 0.8429 - recall: 0.7397 - precision: 0.6798 - AUROC: 0.9039 - AUPRC: 0.7130 - f1_score: 0.7085 - balanced_accuracy: 0.8092 - specificity: 0.8787 - miss_rate: 0.2603 - fall_out: 0.1213 - mcc: 0.6022 - val_loss: 0.3332 - val_accuracy: 0.8443 - val_recall: 0.7723 - val_precision: 0.6728 - val_AUROC: 0.9057 - val_AUPRC: 0.7189 - val_f1_score: 0.7191 - val_balanced_accuracy: 0.8208 - val_specificity: 0.8693 - val_miss_rate: 0.2277 - val_fall_out: 0.1307 - val_mcc: 0.6149
Epoch 5/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3283 - accuracy: 0.8451 - recall: 0.7412 - precision: 0.6848 - AUROC: 0.9071 - AUPRC: 0.7215 - f1_score: 0.7119 - balanced_accuracy: 0.8112 - specificity: 0.8813 - miss_rate: 0.2588 - fall_out: 0.1187 - mcc: 0.6071 - val_loss: 0.3284 - val_accuracy: 0.8486 - val_recall: 0.7423 - val_precision: 0.6931 - val_AUROC: 0.9083 - val_AUPRC: 0.7260 - val_f1_score: 0.7169 - val_balanced_accuracy: 0.8140 - val_specificity: 0.8856 - val_miss_rate: 0.2577 - val_fall_out: 0.1144 - val_mcc: 0.6144
Epoch 6/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3252 - accuracy: 0.8471 - recall: 0.7498 - precision: 0.6866 - AUROC: 0.9090 - AUPRC: 0.7246 - f1_score: 0.7168 - balanced_accuracy: 0.8154 - specificity: 0.8809 - miss_rate: 0.2502 - fall_out: 0.1191 - mcc: 0.6135 - val_loss: 0.3277 - val_accuracy: 0.8480 - val_recall: 0.7646 - val_precision: 0.6838 - val_AUROC: 0.9106 - val_AUPRC: 0.7345 - val_f1_score: 0.7220 - val_balanced_accuracy: 0.8208 - val_specificity: 0.8770 - val_miss_rate: 0.2354 - val_fall_out: 0.1230 - val_mcc: 0.6196
Epoch 7/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3215 - accuracy: 0.8497 - recall: 0.7549 - precision: 0.6914 - AUROC: 0.9113 - AUPRC: 0.7306 - f1_score: 0.7217 - balanced_accuracy: 0.8188 - specificity: 0.8827 - miss_rate: 0.2451 - fall_out: 0.1173 - mcc: 0.6202 - val_loss: 0.3241 - val_accuracy: 0.8487 - val_recall: 0.7345 - val_precision: 0.6962 - val_AUROC: 0.9110 - val_AUPRC: 0.7334 - val_f1_score: 0.7149 - val_balanced_accuracy: 0.8115 - val_specificity: 0.8885 - val_miss_rate: 0.2655 - val_fall_out: 0.1115 - val_mcc: 0.6124
Epoch 8/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3178 - accuracy: 0.8504 - recall: 0.7556 - precision: 0.6928 - AUROC: 0.9134 - AUPRC: 0.7385 - f1_score: 0.7228 - balanced_accuracy: 0.8195 - specificity: 0.8834 - miss_rate: 0.2444 - fall_out: 0.1166 - mcc: 0.6218 - val_loss: 0.3240 - val_accuracy: 0.8494 - val_recall: 0.7128 - val_precision: 0.7064 - val_AUROC: 0.9122 - val_AUPRC: 0.7412 - val_f1_score: 0.7096 - val_balanced_accuracy: 0.8049 - val_specificity: 0.8969 - val_miss_rate: 0.2872 - val_fall_out: 0.1031 - val_mcc: 0.6079
Epoch 9/1000
313/313 [==============================] - 13s 40ms/step - loss: 0.3139 - accuracy: 0.8523 - recall: 0.7561 - precision: 0.6972 - AUROC: 0.9161 - AUPRC: 0.7486 - f1_score: 0.7255 - balanced_accuracy: 0.8209 - specificity: 0.8858 - miss_rate: 0.2439 - fall_out: 0.1142 - mcc: 0.6256 - val_loss: 0.3266 - val_accuracy: 0.8517 - val_recall: 0.7266 - val_precision: 0.7070 - val_AUROC: 0.9112 - val_AUPRC: 0.7395 - val_f1_score: 0.7166 - val_balanced_accuracy: 0.8109 - val_specificity: 0.8952 - val_miss_rate: 0.2734 - val_fall_out: 0.1048 - val_mcc: 0.6163
Epoch 10/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3100 - accuracy: 0.8547 - recall: 0.7561 - precision: 0.7033 - AUROC: 0.9181 - AUPRC: 0.7567 - f1_score: 0.7287 - balanced_accuracy: 0.8226 - specificity: 0.8890 - miss_rate: 0.2439 - fall_out: 0.1110 - mcc: 0.6305 - val_loss: 0.3189 - val_accuracy: 0.8547 - val_recall: 0.7272 - val_precision: 0.7148 - val_AUROC: 0.9153 - val_AUPRC: 0.7511 - val_f1_score: 0.7209 - val_balanced_accuracy: 0.8131 - val_specificity: 0.8991 - val_miss_rate: 0.2728 - val_fall_out: 0.1009 - val_mcc: 0.6228
Epoch 11/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.3057 - accuracy: 0.8587 - recall: 0.7605 - precision: 0.7119 - AUROC: 0.9209 - AUPRC: 0.7651 - f1_score: 0.7354 - balanced_accuracy: 0.8267 - specificity: 0.8929 - miss_rate: 0.2395 - fall_out: 0.1071 - mcc: 0.6399 - val_loss: 0.3182 - val_accuracy: 0.8524 - val_recall: 0.7301 - val_precision: 0.7076 - val_AUROC: 0.9148 - val_AUPRC: 0.7521 - val_f1_score: 0.7186 - val_balanced_accuracy: 0.8125 - val_specificity: 0.8950 - val_miss_rate: 0.2699 - val_fall_out: 0.1050 - val_mcc: 0.6188
Epoch 12/1000
313/313 [==============================] - 14s 42ms/step - loss: 0.3003 - accuracy: 0.8603 - recall: 0.7564 - precision: 0.7178 - AUROC: 0.9241 - AUPRC: 0.7790 - f1_score: 0.7366 - balanced_accuracy: 0.8264 - specificity: 0.8965 - miss_rate: 0.2436 - fall_out: 0.1035 - mcc: 0.6420 - val_loss: 0.3189 - val_accuracy: 0.8527 - val_recall: 0.7089 - val_precision: 0.7171 - val_AUROC: 0.9149 - val_AUPRC: 0.7527 - val_f1_score: 0.7130 - val_balanced_accuracy: 0.8058 - val_specificity: 0.9027 - val_miss_rate: 0.2911 - val_fall_out: 0.0973 - val_mcc: 0.6139
Epoch 13/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2953 - accuracy: 0.8626 - recall: 0.7540 - precision: 0.7247 - AUROC: 0.9268 - AUPRC: 0.7894 - f1_score: 0.7391 - balanced_accuracy: 0.8272 - specificity: 0.9004 - miss_rate: 0.2460 - fall_out: 0.0996 - mcc: 0.6461 - val_loss: 0.3159 - val_accuracy: 0.8525 - val_recall: 0.7359 - val_precision: 0.7054 - val_AUROC: 0.9155 - val_AUPRC: 0.7489 - val_f1_score: 0.7203 - val_balanced_accuracy: 0.8145 - val_specificity: 0.8930 - val_miss_rate: 0.2641 - val_fall_out: 0.1070 - val_mcc: 0.6205
Epoch 14/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2890 - accuracy: 0.8672 - recall: 0.7568 - precision: 0.7361 - AUROC: 0.9304 - AUPRC: 0.8020 - f1_score: 0.7463 - balanced_accuracy: 0.8312 - specificity: 0.9056 - miss_rate: 0.2432 - fall_out: 0.0944 - mcc: 0.6565 - val_loss: 0.3150 - val_accuracy: 0.8553 - val_recall: 0.7148 - val_precision: 0.7220 - val_AUROC: 0.9161 - val_AUPRC: 0.7579 - val_f1_score: 0.7184 - val_balanced_accuracy: 0.8095 - val_specificity: 0.9043 - val_miss_rate: 0.2852 - val_fall_out: 0.0957 - val_mcc: 0.6211
Epoch 15/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2830 - accuracy: 0.8715 - recall: 0.7612 - precision: 0.7462 - AUROC: 0.9335 - AUPRC: 0.8132 - f1_score: 0.7536 - balanced_accuracy: 0.8355 - specificity: 0.9099 - miss_rate: 0.2388 - fall_out: 0.0901 - mcc: 0.6668 - val_loss: 0.3154 - val_accuracy: 0.8528 - val_recall: 0.6764 - val_precision: 0.7328 - val_AUROC: 0.9168 - val_AUPRC: 0.7674 - val_f1_score: 0.7034 - val_balanced_accuracy: 0.7953 - val_specificity: 0.9142 - val_miss_rate: 0.3236 - val_fall_out: 0.0858 - val_mcc: 0.6066
Epoch 16/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2749 - accuracy: 0.8767 - recall: 0.7640 - precision: 0.7597 - AUROC: 0.9376 - AUPRC: 0.8265 - f1_score: 0.7618 - balanced_accuracy: 0.8399 - specificity: 0.9159 - miss_rate: 0.2360 - fall_out: 0.0841 - mcc: 0.6787 - val_loss: 0.3148 - val_accuracy: 0.8547 - val_recall: 0.6942 - val_precision: 0.7299 - val_AUROC: 0.9168 - val_AUPRC: 0.7634 - val_f1_score: 0.7116 - val_balanced_accuracy: 0.8024 - val_specificity: 0.9106 - val_miss_rate: 0.3058 - val_fall_out: 0.0894 - val_mcc: 0.6149
Epoch 17/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2681 - accuracy: 0.8789 - recall: 0.7565 - precision: 0.7703 - AUROC: 0.9408 - AUPRC: 0.8391 - f1_score: 0.7633 - balanced_accuracy: 0.8390 - specificity: 0.9215 - miss_rate: 0.2435 - fall_out: 0.0785 - mcc: 0.6820 - val_loss: 0.3174 - val_accuracy: 0.8539 - val_recall: 0.7301 - val_precision: 0.7116 - val_AUROC: 0.9157 - val_AUPRC: 0.7595 - val_f1_score: 0.7207 - val_balanced_accuracy: 0.8136 - val_specificity: 0.8970 - val_miss_rate: 0.2699 - val_fall_out: 0.1030 - val_mcc: 0.6219
Epoch 18/1000
313/313 [==============================] - 13s 41ms/step - loss: 0.2626 - accuracy: 0.8833 - recall: 0.7689 - precision: 0.7768 - AUROC: 0.9433 - AUPRC: 0.8464 - f1_score: 0.7729 - balanced_accuracy: 0.8460 - specificity: 0.9231 - miss_rate: 0.2311 - fall_out: 0.0769 - mcc: 0.6944 - val_loss: 0.3237 - val_accuracy: 0.8451 - val_recall: 0.6290 - val_precision: 0.7329 - val_AUROC: 0.9145 - val_AUPRC: 0.7566 - val_f1_score: 0.6770 - val_balanced_accuracy: 0.7746 - val_specificity: 0.9202 - val_miss_rate: 0.3710 - val_fall_out: 0.0798 - val_mcc: 0.5788
313/313 [==============================] - 8s 26ms/step - loss: 0.2250 - accuracy: 0.9079 - recall: 0.7490 - precision: 0.8761 - AUROC: 0.9636 - AUPRC: 0.9076 - f1_score: 0.8076 - balanced_accuracy: 0.8561 - specificity: 0.9632 - miss_rate: 0.2510 - fall_out: 0.0368 - mcc: 0.7515
79/79 [==============================] - 4s 52ms/step - loss: 0.3237 - accuracy: 0.8451 - recall: 0.6290 - precision: 0.7329 - AUROC: 0.9145 - AUPRC: 0.7566 - f1_score: 0.6770 - balanced_accuracy: 0.7746 - specificity: 0.9202 - miss_rate: 0.3710 - fall_out: 0.0798 - mcc: 0.5788
-- HOLDOUT 10 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['ZNF207', 'ZNF737']
-- Actually uncorrelated features: [confirmed via MIC]
['ZNF207', 'ZNF737']
-- 1 Correlated features: [Pearson]
['SAP130']
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 560)] 0
)
dense (Dense) (None, 128) 71808
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,289
Trainable params: 84,289
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 560)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71808 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 211,873
Trainable params: 211,873
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 20s 55ms/step - loss: 11.4489 - accuracy: 0.7980 - recall: 0.5430 - precision: 0.6251 - AUROC: 0.8562 - AUPRC: 0.6175 - f1_score: 0.5812 - balanced_accuracy: 0.7149 - specificity: 0.8867 - miss_rate: 0.4570 - fall_out: 0.1133 - mcc: 0.4509 - val_loss: 2.2611 - val_accuracy: 0.8061 - val_recall: 0.3925 - val_precision: 0.7320 - val_AUROC: 0.8995 - val_AUPRC: 0.6868 - val_f1_score: 0.5110 - val_balanced_accuracy: 0.6712 - val_specificity: 0.9500 - val_miss_rate: 0.6075 - val_fall_out: 0.0500 - val_mcc: 0.4340
Epoch 2/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.9401 - accuracy: 0.8298 - recall: 0.7209 - precision: 0.6546 - AUROC: 0.8898 - AUPRC: 0.6752 - f1_score: 0.6861 - balanced_accuracy: 0.7943 - specificity: 0.8676 - miss_rate: 0.2791 - fall_out: 0.1324 - mcc: 0.5710 - val_loss: 1.6353 - val_accuracy: 0.8438 - val_recall: 0.7671 - val_precision: 0.6734 - val_AUROC: 0.9050 - val_AUPRC: 0.7053 - val_f1_score: 0.7172 - val_balanced_accuracy: 0.8188 - val_specificity: 0.8705 - val_miss_rate: 0.2329 - val_fall_out: 0.1295 - val_mcc: 0.6124
Epoch 3/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.7510 - accuracy: 0.8348 - recall: 0.7487 - precision: 0.6583 - AUROC: 0.8952 - AUPRC: 0.6877 - f1_score: 0.7006 - balanced_accuracy: 0.8067 - specificity: 0.8648 - miss_rate: 0.2513 - fall_out: 0.1352 - mcc: 0.5895 - val_loss: 1.8295 - val_accuracy: 0.8454 - val_recall: 0.7776 - val_precision: 0.6738 - val_AUROC: 0.9067 - val_AUPRC: 0.7107 - val_f1_score: 0.7220 - val_balanced_accuracy: 0.8233 - val_specificity: 0.8690 - val_miss_rate: 0.2224 - val_fall_out: 0.1310 - val_mcc: 0.6187
Epoch 4/1000
313/313 [==============================] - 18s 56ms/step - loss: 1.6261 - accuracy: 0.8377 - recall: 0.7562 - precision: 0.6627 - AUROC: 0.8984 - AUPRC: 0.6958 - f1_score: 0.7064 - balanced_accuracy: 0.8112 - specificity: 0.8661 - miss_rate: 0.2438 - fall_out: 0.1339 - mcc: 0.5974 - val_loss: 1.3706 - val_accuracy: 0.8474 - val_recall: 0.7687 - val_precision: 0.6811 - val_AUROC: 0.9092 - val_AUPRC: 0.7175 - val_f1_score: 0.7222 - val_balanced_accuracy: 0.8217 - val_specificity: 0.8748 - val_miss_rate: 0.2313 - val_fall_out: 0.1252 - val_mcc: 0.6197
Epoch 5/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.5337 - accuracy: 0.8400 - recall: 0.7621 - precision: 0.6661 - AUROC: 0.9010 - AUPRC: 0.7024 - f1_score: 0.7109 - balanced_accuracy: 0.8146 - specificity: 0.8671 - miss_rate: 0.2379 - fall_out: 0.1329 - mcc: 0.6035 - val_loss: 1.7155 - val_accuracy: 0.8471 - val_recall: 0.7848 - val_precision: 0.6754 - val_AUROC: 0.9096 - val_AUPRC: 0.7193 - val_f1_score: 0.7260 - val_balanced_accuracy: 0.8268 - val_specificity: 0.8688 - val_miss_rate: 0.2152 - val_fall_out: 0.1312 - val_mcc: 0.6241
Epoch 6/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.4652 - accuracy: 0.8423 - recall: 0.7689 - precision: 0.6694 - AUROC: 0.9035 - AUPRC: 0.7080 - f1_score: 0.7157 - balanced_accuracy: 0.8184 - specificity: 0.8679 - miss_rate: 0.2311 - fall_out: 0.1321 - mcc: 0.6102 - val_loss: 1.2549 - val_accuracy: 0.8518 - val_recall: 0.7283 - val_precision: 0.7065 - val_AUROC: 0.9117 - val_AUPRC: 0.7252 - val_f1_score: 0.7173 - val_balanced_accuracy: 0.8115 - val_specificity: 0.8947 - val_miss_rate: 0.2717 - val_fall_out: 0.1053 - val_mcc: 0.6170
Epoch 7/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.4098 - accuracy: 0.8442 - recall: 0.7628 - precision: 0.6755 - AUROC: 0.9050 - AUPRC: 0.7112 - f1_score: 0.7165 - balanced_accuracy: 0.8177 - specificity: 0.8725 - miss_rate: 0.2372 - fall_out: 0.1275 - mcc: 0.6118 - val_loss: 1.5431 - val_accuracy: 0.8493 - val_recall: 0.7735 - val_precision: 0.6840 - val_AUROC: 0.9116 - val_AUPRC: 0.7270 - val_f1_score: 0.7260 - val_balanced_accuracy: 0.8246 - val_specificity: 0.8756 - val_miss_rate: 0.2265 - val_fall_out: 0.1244 - val_mcc: 0.6248
Epoch 8/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.3619 - accuracy: 0.8458 - recall: 0.7622 - precision: 0.6794 - AUROC: 0.9070 - AUPRC: 0.7167 - f1_score: 0.7184 - balanced_accuracy: 0.8185 - specificity: 0.8748 - miss_rate: 0.2378 - fall_out: 0.1252 - mcc: 0.6146 - val_loss: 1.1767 - val_accuracy: 0.8506 - val_recall: 0.7148 - val_precision: 0.7088 - val_AUROC: 0.9130 - val_AUPRC: 0.7303 - val_f1_score: 0.7118 - val_balanced_accuracy: 0.8063 - val_specificity: 0.8978 - val_miss_rate: 0.2852 - val_fall_out: 0.1022 - val_mcc: 0.6109
Epoch 9/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.3206 - accuracy: 0.8453 - recall: 0.7607 - precision: 0.6788 - AUROC: 0.9080 - AUPRC: 0.7212 - f1_score: 0.7174 - balanced_accuracy: 0.8177 - specificity: 0.8748 - miss_rate: 0.2393 - fall_out: 0.1252 - mcc: 0.6133 - val_loss: 1.4282 - val_accuracy: 0.8517 - val_recall: 0.7351 - val_precision: 0.7037 - val_AUROC: 0.9134 - val_AUPRC: 0.7330 - val_f1_score: 0.7191 - val_balanced_accuracy: 0.8137 - val_specificity: 0.8923 - val_miss_rate: 0.2649 - val_fall_out: 0.1077 - val_mcc: 0.6187
Epoch 10/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.2841 - accuracy: 0.8466 - recall: 0.7628 - precision: 0.6810 - AUROC: 0.9095 - AUPRC: 0.7240 - f1_score: 0.7196 - balanced_accuracy: 0.8192 - specificity: 0.8757 - miss_rate: 0.2372 - fall_out: 0.1243 - mcc: 0.6163 - val_loss: 1.1516 - val_accuracy: 0.8517 - val_recall: 0.6973 - val_precision: 0.7195 - val_AUROC: 0.9140 - val_AUPRC: 0.7350 - val_f1_score: 0.7082 - val_balanced_accuracy: 0.8014 - val_specificity: 0.9054 - val_miss_rate: 0.3027 - val_fall_out: 0.0946 - val_mcc: 0.6090
Epoch 11/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.2483 - accuracy: 0.8485 - recall: 0.7604 - precision: 0.6865 - AUROC: 0.9105 - AUPRC: 0.7261 - f1_score: 0.7216 - balanced_accuracy: 0.8198 - specificity: 0.8792 - miss_rate: 0.2396 - fall_out: 0.1208 - mcc: 0.6195 - val_loss: 1.3327 - val_accuracy: 0.8514 - val_recall: 0.6961 - val_precision: 0.7193 - val_AUROC: 0.9142 - val_AUPRC: 0.7345 - val_f1_score: 0.7075 - val_balanced_accuracy: 0.8008 - val_specificity: 0.9055 - val_miss_rate: 0.3039 - val_fall_out: 0.0945 - val_mcc: 0.6081
Epoch 12/1000
313/313 [==============================] - 17s 53ms/step - loss: 1.2175 - accuracy: 0.8511 - recall: 0.7673 - precision: 0.6903 - AUROC: 0.9117 - AUPRC: 0.7293 - f1_score: 0.7268 - balanced_accuracy: 0.8238 - specificity: 0.8802 - miss_rate: 0.2327 - fall_out: 0.1198 - mcc: 0.6265 - val_loss: 1.1518 - val_accuracy: 0.8527 - val_recall: 0.7553 - val_precision: 0.6985 - val_AUROC: 0.9149 - val_AUPRC: 0.7384 - val_f1_score: 0.7258 - val_balanced_accuracy: 0.8209 - val_specificity: 0.8866 - val_miss_rate: 0.2447 - val_fall_out: 0.1134 - val_mcc: 0.6262
313/313 [==============================] - 9s 28ms/step - loss: 1.1406 - accuracy: 0.8582 - recall: 0.7711 - precision: 0.7064 - AUROC: 0.9219 - AUPRC: 0.7565 - f1_score: 0.7373 - balanced_accuracy: 0.8298 - specificity: 0.8885 - miss_rate: 0.2289 - fall_out: 0.1115 - mcc: 0.6416
79/79 [==============================] - 5s 57ms/step - loss: 1.1518 - accuracy: 0.8527 - recall: 0.7553 - precision: 0.6985 - AUROC: 0.9149 - AUPRC: 0.7384 - f1_score: 0.7258 - balanced_accuracy: 0.8209 - specificity: 0.8866 - miss_rate: 0.2447 - fall_out: 0.1134 - mcc: 0.6262
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_promoters\MLP_hypermodel_promoters_9\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_promoters\MLP_hypermodel_promoters_9\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 104, 'dropout_1': 0.3, 'units_2': 104, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.5, 'units_4': 72, 'dropout_4': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_promoters\CNN_hypermodel_promoters_9\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_promoters\CNN_hypermodel_promoters_9\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 3, 'filters_0': 144, 'kernel_size_0': 6, 'conv_dropout_0': 0.0, 'dense_depth': 1, 'units_0': 16, 'dense_dropout_0': 0.5, 'filters_1': 32, 'kernel_size_1': 8, 'conv_dropout_1': 0.2, 'filters_2': 104, 'kernel_size_2': 8, 'conv_dropout_2': 0.2, 'units_1': 16, 'dense_dropout_1': 0.5, 'units_2': 16, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.3}
- Removing uncorrelated/correlated features: ['ZNF207', 'ZNF737', 'SAP130']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 1) 137
=================================================================
Total params: 76,569
Trainable params: 76,569
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 144) 3600
dropout_4 (Dropout) (None, 256, 144) 0
max_pooling1d_1 (MaxPooling (None, 128, 144) 0
1D)
conv1d_2 (Conv1D) (None, 128, 32) 36896
dropout_5 (Dropout) (None, 128, 32) 0
max_pooling1d_2 (MaxPooling (None, 64, 32) 0
1D)
conv1d_3 (Conv1D) (None, 64, 104) 26728
dropout_6 (Dropout) (None, 64, 104) 0
max_pooling1d_3 (MaxPooling (None, 32, 104) 0
1D)
flatten_1 (Flatten) (None, 3328) 0
dense_6 (Dense) (None, 16) 53264
dropout_7 (Dropout) (None, 16) 0
dense_7 (Dense) (None, 1) 17
=================================================================
Total params: 120,505
Trainable params: 120,505
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 144) 3600 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 144) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 144) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 32) 36896 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 104) 26728 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 104) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 104) 0 ['dropout_6[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
flatten_1 (Flatten) (None, 3328) 0 ['max_pooling1d_3[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
dense_6 (Dense) (None, 16) 53264 ['flatten_1[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_7 (Dropout) (None, 16) 0 ['dense_6[0][0]']
concatenate (Concatenate) (None, 152) 0 ['dropout_3[0][0]',
'dropout_7[0][0]']
dense_8 (Dense) (None, 64) 9792 ['concatenate[0][0]']
dense_9 (Dense) (None, 1) 65 ['dense_8[0][0]']
==================================================================================================
Total params: 206,777
Trainable params: 206,777
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
313/313 [==============================] - 21s 52ms/step - loss: 0.3747 - accuracy: 0.8224 - recall: 0.6694 - precision: 0.6519 - AUROC: 0.8802 - AUPRC: 0.6608 - f1_score: 0.6606 - balanced_accuracy: 0.7725 - specificity: 0.8756 - miss_rate: 0.3306 - fall_out: 0.1244 - mcc: 0.5404 - val_loss: 0.3358 - val_accuracy: 0.8419 - val_recall: 0.7568 - val_precision: 0.6721 - val_AUROC: 0.9033 - val_AUPRC: 0.6999 - val_f1_score: 0.7120 - val_balanced_accuracy: 0.8142 - val_specificity: 0.8715 - val_miss_rate: 0.2432 - val_fall_out: 0.1285 - val_mcc: 0.6056
Epoch 2/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.3398 - accuracy: 0.8397 - recall: 0.7263 - precision: 0.6765 - AUROC: 0.8999 - AUPRC: 0.7019 - f1_score: 0.7005 - balanced_accuracy: 0.8027 - specificity: 0.8791 - miss_rate: 0.2737 - fall_out: 0.1209 - mcc: 0.5919 - val_loss: 0.3298 - val_accuracy: 0.8482 - val_recall: 0.7378 - val_precision: 0.6937 - val_AUROC: 0.9080 - val_AUPRC: 0.7151 - val_f1_score: 0.7151 - val_balanced_accuracy: 0.8122 - val_specificity: 0.8866 - val_miss_rate: 0.2622 - val_fall_out: 0.1134 - val_mcc: 0.6123
Epoch 3/1000
313/313 [==============================] - 12s 37ms/step - loss: 0.3292 - accuracy: 0.8453 - recall: 0.7429 - precision: 0.6846 - AUROC: 0.9067 - AUPRC: 0.7222 - f1_score: 0.7126 - balanced_accuracy: 0.8119 - specificity: 0.8809 - miss_rate: 0.2571 - fall_out: 0.1191 - mcc: 0.6079 - val_loss: 0.3244 - val_accuracy: 0.8513 - val_recall: 0.7588 - val_precision: 0.6938 - val_AUROC: 0.9113 - val_AUPRC: 0.7244 - val_f1_score: 0.7248 - val_balanced_accuracy: 0.8211 - val_specificity: 0.8835 - val_miss_rate: 0.2412 - val_fall_out: 0.1165 - val_mcc: 0.6244
Epoch 4/1000
313/313 [==============================] - 12s 37ms/step - loss: 0.3223 - accuracy: 0.8483 - recall: 0.7422 - precision: 0.6923 - AUROC: 0.9110 - AUPRC: 0.7347 - f1_score: 0.7164 - balanced_accuracy: 0.8137 - specificity: 0.8852 - miss_rate: 0.2578 - fall_out: 0.1148 - mcc: 0.6137 - val_loss: 0.3201 - val_accuracy: 0.8526 - val_recall: 0.7281 - val_precision: 0.7088 - val_AUROC: 0.9141 - val_AUPRC: 0.7405 - val_f1_score: 0.7183 - val_balanced_accuracy: 0.8120 - val_specificity: 0.8959 - val_miss_rate: 0.2719 - val_fall_out: 0.1041 - val_mcc: 0.6186
Epoch 5/1000
313/313 [==============================] - 12s 37ms/step - loss: 0.3155 - accuracy: 0.8527 - recall: 0.7472 - precision: 0.7016 - AUROC: 0.9152 - AUPRC: 0.7478 - f1_score: 0.7237 - balanced_accuracy: 0.8183 - specificity: 0.8894 - miss_rate: 0.2528 - fall_out: 0.1106 - mcc: 0.6240 - val_loss: 0.3176 - val_accuracy: 0.8539 - val_recall: 0.6775 - val_precision: 0.7357 - val_AUROC: 0.9172 - val_AUPRC: 0.7535 - val_f1_score: 0.7054 - val_balanced_accuracy: 0.7964 - val_specificity: 0.9153 - val_miss_rate: 0.3225 - val_fall_out: 0.0847 - val_mcc: 0.6095
Epoch 6/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.3082 - accuracy: 0.8559 - recall: 0.7484 - precision: 0.7094 - AUROC: 0.9197 - AUPRC: 0.7645 - f1_score: 0.7284 - balanced_accuracy: 0.8208 - specificity: 0.8933 - miss_rate: 0.2516 - fall_out: 0.1067 - mcc: 0.6308 - val_loss: 0.3134 - val_accuracy: 0.8583 - val_recall: 0.7198 - val_precision: 0.7281 - val_AUROC: 0.9184 - val_AUPRC: 0.7516 - val_f1_score: 0.7239 - val_balanced_accuracy: 0.8131 - val_specificity: 0.9065 - val_miss_rate: 0.2802 - val_fall_out: 0.0935 - val_mcc: 0.6286
Epoch 7/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.3019 - accuracy: 0.8604 - recall: 0.7532 - precision: 0.7193 - AUROC: 0.9234 - AUPRC: 0.7737 - f1_score: 0.7358 - balanced_accuracy: 0.8255 - specificity: 0.8977 - miss_rate: 0.2468 - fall_out: 0.1023 - mcc: 0.6414 - val_loss: 0.3093 - val_accuracy: 0.8591 - val_recall: 0.7307 - val_precision: 0.7255 - val_AUROC: 0.9204 - val_AUPRC: 0.7591 - val_f1_score: 0.7280 - val_balanced_accuracy: 0.8172 - val_specificity: 0.9038 - val_miss_rate: 0.2693 - val_fall_out: 0.0962 - val_mcc: 0.6330
Epoch 8/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.2952 - accuracy: 0.8638 - recall: 0.7555 - precision: 0.7275 - AUROC: 0.9268 - AUPRC: 0.7898 - f1_score: 0.7412 - balanced_accuracy: 0.8285 - specificity: 0.9015 - miss_rate: 0.2445 - fall_out: 0.0985 - mcc: 0.6491 - val_loss: 0.3097 - val_accuracy: 0.8589 - val_recall: 0.7574 - val_precision: 0.7136 - val_AUROC: 0.9195 - val_AUPRC: 0.7571 - val_f1_score: 0.7348 - val_balanced_accuracy: 0.8258 - val_specificity: 0.8942 - val_miss_rate: 0.2426 - val_fall_out: 0.1058 - val_mcc: 0.6393
Epoch 9/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.2887 - accuracy: 0.8662 - recall: 0.7556 - precision: 0.7340 - AUROC: 0.9305 - AUPRC: 0.7999 - f1_score: 0.7446 - balanced_accuracy: 0.8302 - specificity: 0.9047 - miss_rate: 0.2444 - fall_out: 0.0953 - mcc: 0.6542 - val_loss: 0.3063 - val_accuracy: 0.8607 - val_recall: 0.7169 - val_precision: 0.7366 - val_AUROC: 0.9222 - val_AUPRC: 0.7735 - val_f1_score: 0.7266 - val_balanced_accuracy: 0.8138 - val_specificity: 0.9108 - val_miss_rate: 0.2831 - val_fall_out: 0.0892 - val_mcc: 0.6333
Epoch 10/1000
313/313 [==============================] - 12s 37ms/step - loss: 0.2834 - accuracy: 0.8713 - recall: 0.7620 - precision: 0.7453 - AUROC: 0.9332 - AUPRC: 0.8096 - f1_score: 0.7535 - balanced_accuracy: 0.8357 - specificity: 0.9094 - miss_rate: 0.2380 - fall_out: 0.0906 - mcc: 0.6666 - val_loss: 0.3065 - val_accuracy: 0.8600 - val_recall: 0.7456 - val_precision: 0.7215 - val_AUROC: 0.9215 - val_AUPRC: 0.7699 - val_f1_score: 0.7334 - val_balanced_accuracy: 0.8227 - val_specificity: 0.8999 - val_miss_rate: 0.2544 - val_fall_out: 0.1001 - val_mcc: 0.6387
Epoch 11/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.2753 - accuracy: 0.8750 - recall: 0.7675 - precision: 0.7530 - AUROC: 0.9373 - AUPRC: 0.8237 - f1_score: 0.7602 - balanced_accuracy: 0.8399 - specificity: 0.9124 - miss_rate: 0.2325 - fall_out: 0.0876 - mcc: 0.6757 - val_loss: 0.3050 - val_accuracy: 0.8586 - val_recall: 0.7246 - val_precision: 0.7269 - val_AUROC: 0.9218 - val_AUPRC: 0.7751 - val_f1_score: 0.7258 - val_balanced_accuracy: 0.8150 - val_specificity: 0.9053 - val_miss_rate: 0.2754 - val_fall_out: 0.0947 - val_mcc: 0.6305
Epoch 12/1000
313/313 [==============================] - 12s 37ms/step - loss: 0.2695 - accuracy: 0.8784 - recall: 0.7691 - precision: 0.7619 - AUROC: 0.9403 - AUPRC: 0.8327 - f1_score: 0.7655 - balanced_accuracy: 0.8428 - specificity: 0.9164 - miss_rate: 0.2309 - fall_out: 0.0836 - mcc: 0.6834 - val_loss: 0.3167 - val_accuracy: 0.8572 - val_recall: 0.7082 - val_precision: 0.7305 - val_AUROC: 0.9193 - val_AUPRC: 0.7682 - val_f1_score: 0.7192 - val_balanced_accuracy: 0.8086 - val_specificity: 0.9091 - val_miss_rate: 0.2918 - val_fall_out: 0.0909 - val_mcc: 0.6236
Epoch 13/1000
313/313 [==============================] - 12s 36ms/step - loss: 0.2639 - accuracy: 0.8814 - recall: 0.7767 - precision: 0.7668 - AUROC: 0.9427 - AUPRC: 0.8404 - f1_score: 0.7717 - balanced_accuracy: 0.8473 - specificity: 0.9178 - miss_rate: 0.2233 - fall_out: 0.0822 - mcc: 0.6916 - val_loss: 0.3061 - val_accuracy: 0.8605 - val_recall: 0.7338 - val_precision: 0.7281 - val_AUROC: 0.9220 - val_AUPRC: 0.7737 - val_f1_score: 0.7309 - val_balanced_accuracy: 0.8192 - val_specificity: 0.9047 - val_miss_rate: 0.2662 - val_fall_out: 0.0953 - val_mcc: 0.6368
313/313 [==============================] - 8s 25ms/step - loss: 0.2304 - accuracy: 0.9034 - recall: 0.8193 - precision: 0.8091 - AUROC: 0.9599 - AUPRC: 0.8917 - f1_score: 0.8141 - balanced_accuracy: 0.8760 - specificity: 0.9327 - miss_rate: 0.1807 - fall_out: 0.0673 - mcc: 0.7490
79/79 [==============================] - 5s 59ms/step - loss: 0.3061 - accuracy: 0.8605 - recall: 0.7338 - precision: 0.7281 - AUROC: 0.9220 - AUPRC: 0.7737 - f1_score: 0.7309 - balanced_accuracy: 0.8192 - specificity: 0.9047 - miss_rate: 0.2662 - fall_out: 0.0953 - mcc: 0.6368
MMNN_metrics_estimate = model_metrics_holdout_estimate_evaluation(MMNN_metrics_promoters["tuning_False"], number_of_splits)
print("-- PROMOTERS -- with FIXED MLP and CNN")
print(f"MMNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {MMNN_metrics_estimate['accuracy_train']} -- test - {MMNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {MMNN_metrics_estimate['AUROC_train']} -- test - {MMNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {MMNN_metrics_estimate['AUPRC_train']} -- test - {MMNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("MMNN - Train history:")
plot_train_history(MMNN_history_promoters["tuning_False"])
print("-"*100)
MMNN_metrics_estimate = model_metrics_holdout_estimate_evaluation(MMNN_metrics_promoters["tuning_True"], number_of_splits)
print("-- PROMOTERS -- with bayesian optimized hyperparameters MLP and CNN")
print(f"MMNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {MMNN_metrics_estimate['accuracy_train']} -- test - {MMNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {MMNN_metrics_estimate['AUROC_train']} -- test - {MMNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {MMNN_metrics_estimate['AUPRC_train']} -- test - {MMNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("MMNN - Train history:")
plot_train_history(MMNN_history_promoters["tuning_True"])
-- PROMTOTERS -- with FIXED MLP and CNN MMNN Metrics - 10-holdouts estimate: Accuracy : train - 0.8591259539127349 -- test - 0.8498673617839814 AUROC : train - 0.9226655066013336 -- test - 0.9122659921646118 AUPRC : train - 0.757349681854248 -- test - 0.7347532510757446 -------------------------------------------------------------------------------- MMNN - Train history:
---------------------------------------------------------------------------------------------------- -- PROMTOTERS -- with bayesian optimized hyperparameters MLP and CNN MMNN Metrics - 10-holdouts estimate: Accuracy : train - 0.8814577519893646 -- test - 0.8502327740192414 AUROC : train - 0.9431495547294617 -- test - 0.9146451532840729 AUPRC : train - 0.8385738432407379 -- test - 0.7485358357429505 -------------------------------------------------------------------------------- MMNN - Train history:
MMNN_metrics_promoters
{'tuning_True': [{'train_evaluation': {'loss': 0.26318269968032837,
'accuracy': 0.8808194994926453,
'recall': 0.81207275390625,
'precision': 0.7478567361831665,
'AUROC': 0.9440953135490417,
'AUPRC': 0.8309550881385803,
'f1_score': 0.778643012046814,
'balanced_accuracy': 0.8584057092666626,
'specificity': 0.9047386050224304,
'miss_rate': 0.1879272758960724,
'fall_out': 0.09526138752698898,
'mcc': 0.6983979344367981},
'test_evaluation': {'loss': 0.32070329785346985,
'accuracy': 0.8511788845062256,
'recall': 0.7537328004837036,
'precision': 0.695348858833313,
'AUROC': 0.9124017953872681,
'AUPRC': 0.7418513894081116,
'f1_score': 0.7233646512031555,
'balanced_accuracy': 0.8194102644920349,
'specificity': 0.8850877285003662,
'miss_rate': 0.24626721441745758,
'fall_out': 0.11491227895021439,
'mcc': 0.6227453947067261}},
{'train_evaluation': {'loss': 0.2676412761211395,
'accuracy': 0.8855876922607422,
'recall': 0.7583515048027039,
'precision': 0.7899894118309021,
'AUROC': 0.9453145265579224,
'AUPRC': 0.8497247695922852,
'f1_score': 0.773847222328186,
'balanced_accuracy': 0.8441042900085449,
'specificity': 0.9298571348190308,
'miss_rate': 0.24164848029613495,
'fall_out': 0.07014288008213043,
'mcc': 0.6975745558738708},
'test_evaluation': {'loss': 0.32417988777160645,
'accuracy': 0.84877610206604,
'recall': 0.6812099814414978,
'precision': 0.7184048891067505,
'AUROC': 0.9119037389755249,
'AUPRC': 0.7427470088005066,
'f1_score': 0.699313223361969,
'balanced_accuracy': 0.7941474914550781,
'specificity': 0.9070850014686584,
'miss_rate': 0.3187899887561798,
'fall_out': 0.09291497617959976,
'mcc': 0.5987727046012878}},
{'train_evaluation': {'loss': 0.278840035200119,
'accuracy': 0.8717461228370667,
'recall': 0.8497454524040222,
'precision': 0.7102735638618469,
'AUROC': 0.9386857748031616,
'AUPRC': 0.8237740397453308,
'f1_score': 0.7737748622894287,
'balanced_accuracy': 0.8645731210708618,
'specificity': 0.8794007897377014,
'miss_rate': 0.15025454759597778,
'fall_out': 0.12059920281171799,
'mcc': 0.6906370520591736},
'test_evaluation': {'loss': 0.3144226670265198,
'accuracy': 0.8533313274383545,
'recall': 0.8206321597099304,
'precision': 0.6785313487052917,
'AUROC': 0.9179574847221375,
'AUPRC': 0.7553421854972839,
'f1_score': 0.7428470849990845,
'balanced_accuracy': 0.8426710367202759,
'specificity': 0.8647098541259766,
'miss_rate': 0.17936785519123077,
'fall_out': 0.13529014587402344,
'mcc': 0.6472153067588806}},
{'train_evaluation': {'loss': 0.27988481521606445,
'accuracy': 0.8793051838874817,
'recall': 0.6906181573867798,
'precision': 0.8136174082756042,
'AUROC': 0.940905749797821,
'AUPRC': 0.8470730185508728,
'f1_score': 0.7470890879631042,
'balanced_accuracy': 0.817786693572998,
'specificity': 0.9449552297592163,
'miss_rate': 0.3093818128108978,
'fall_out': 0.055044788867235184,
'mcc': 0.6723964214324951},
'test_evaluation': {'loss': 0.3167767822742462,
'accuracy': 0.8466235995292664,
'recall': 0.6218731999397278,
'precision': 0.7421892881393433,
'AUROC': 0.9177714586257935,
'AUPRC': 0.7548065185546875,
'f1_score': 0.6767250299453735,
'balanced_accuracy': 0.7733522653579712,
'specificity': 0.9248313307762146,
'miss_rate': 0.3781268298625946,
'fall_out': 0.07516869157552719,
'mcc': 0.581089198589325}},
{'train_evaluation': {'loss': 0.2916513979434967,
'accuracy': 0.8706072568893433,
'recall': 0.7613090872764587,
'precision': 0.7435362935066223,
'AUROC': 0.9325039386749268,
'AUPRC': 0.8047640919685364,
'f1_score': 0.7523177266120911,
'balanced_accuracy': 0.8349722623825073,
'specificity': 0.9086354374885559,
'miss_rate': 0.23869091272354126,
'fall_out': 0.09136456251144409,
'mcc': 0.664848804473877},
'test_evaluation': {'loss': 0.32179221510887146,
'accuracy': 0.8513290286064148,
'recall': 0.7196044325828552,
'precision': 0.7088825106620789,
'AUROC': 0.912771999835968,
'AUPRC': 0.7403241991996765,
'f1_score': 0.7142032384872437,
'balanced_accuracy': 0.8083852529525757,
'specificity': 0.8971660137176514,
'miss_rate': 0.2803955674171448,
'fall_out': 0.10283400863409042,
'mcc': 0.6137756109237671}},
{'train_evaluation': {'loss': 0.2455703467130661,
'accuracy': 0.8958625197410583,
'recall': 0.7903030514717102,
'precision': 0.8031139373779297,
'AUROC': 0.9553269743919373,
'AUPRC': 0.8772685527801514,
'f1_score': 0.7966569662094116,
'balanced_accuracy': 0.8614464998245239,
'specificity': 0.9325899481773376,
'miss_rate': 0.209696963429451,
'fall_out': 0.06741004437208176,
'mcc': 0.7267146706581116},
'test_evaluation': {'loss': 0.3089958727359772,
'accuracy': 0.8547829985618591,
'recall': 0.7132053375244141,
'precision': 0.7211764454841614,
'AUROC': 0.9196145534515381,
'AUPRC': 0.7678280472755432,
'f1_score': 0.7171687483787537,
'balanced_accuracy': 0.8086269497871399,
'specificity': 0.9040485620498657,
'miss_rate': 0.28679466247558594,
'fall_out': 0.09595141559839249,
'mcc': 0.6195034980773926}},
{'train_evaluation': {'loss': 0.2713947594165802,
'accuracy': 0.8753003478050232,
'recall': 0.8276363611221313,
'precision': 0.7270326614379883,
'AUROC': 0.9408495426177979,
'AUPRC': 0.8237742781639099,
'f1_score': 0.774079442024231,
'balanced_accuracy': 0.8597602844238281,
'specificity': 0.8918841481208801,
'miss_rate': 0.17236363887786865,
'fall_out': 0.10811585932970047,
'mcc': 0.6912178993225098},
'test_evaluation': {'loss': 0.32190337777137756,
'accuracy': 0.8479751944541931,
'recall': 0.7727360725402832,
'precision': 0.6811965703964233,
'AUROC': 0.9124097228050232,
'AUPRC': 0.7414856553077698,
'f1_score': 0.7240846753120422,
'balanced_accuracy': 0.8234462738037109,
'specificity': 0.8741565346717834,
'miss_rate': 0.2272639125585556,
'fall_out': 0.12584345042705536,
'mcc': 0.6220882534980774}},
{'train_evaluation': {'loss': 0.3518398404121399,
'accuracy': 0.8440253138542175,
'recall': 0.8076121211051941,
'precision': 0.6622534990310669,
'AUROC': 0.9103205800056458,
'AUPRC': 0.7291437387466431,
'f1_score': 0.7277453541755676,
'balanced_accuracy': 0.8321533799171448,
'specificity': 0.8566946387290955,
'miss_rate': 0.1923878788948059,
'fall_out': 0.14330539107322693,
'mcc': 0.6259358525276184},
'test_evaluation': {'loss': 0.35547134280204773,
'accuracy': 0.8427191376686096,
'recall': 0.8006592988967896,
'precision': 0.6613807678222656,
'AUROC': 0.9051212072372437,
'AUPRC': 0.7107263803482056,
'f1_score': 0.7243859767913818,
'balanced_accuracy': 0.8290071487426758,
'specificity': 0.8573549389839172,
'miss_rate': 0.19934070110321045,
'fall_out': 0.14264507591724396,
'mcc': 0.6212447881698608}},
{'train_evaluation': {'loss': 0.22503067553043365,
'accuracy': 0.9078769683837891,
'recall': 0.7489939332008362,
'precision': 0.8761342763900757,
'AUROC': 0.9635559320449829,
'AUPRC': 0.9075829386711121,
'f1_score': 0.8075907826423645,
'balanced_accuracy': 0.856075644493103,
'specificity': 0.9631572961807251,
'miss_rate': 0.2510060667991638,
'fall_out': 0.03684272617101669,
'mcc': 0.7514887452125549},
'test_evaluation': {'loss': 0.32369884848594666,
'accuracy': 0.8450718522071838,
'recall': 0.6290478706359863,
'precision': 0.7329416871070862,
'AUROC': 0.9145113825798035,
'AUPRC': 0.7565793395042419,
'f1_score': 0.6770322322845459,
'balanced_accuracy': 0.7746453881263733,
'specificity': 0.9202429056167603,
'miss_rate': 0.3709520995616913,
'fall_out': 0.07975708693265915,
'mcc': 0.5788154602050781}},
{'train_evaluation': {'loss': 0.23042787611484528,
'accuracy': 0.903446614742279,
'recall': 0.8192969560623169,
'precision': 0.8090587258338928,
'AUROC': 0.9599372148513794,
'AUPRC': 0.8916779160499573,
'f1_score': 0.8141456246376038,
'balanced_accuracy': 0.8760108947753906,
'specificity': 0.9327248930931091,
'miss_rate': 0.1807030290365219,
'fall_out': 0.06727509200572968,
'mcc': 0.748958945274353},
'test_evaluation': {'loss': 0.30608025193214417,
'accuracy': 0.8605396151542664,
'recall': 0.7337599396705627,
'precision': 0.7281123995780945,
'AUROC': 0.9219881892204285,
'AUPRC': 0.7736676335334778,
'f1_score': 0.7309252619743347,
'balanced_accuracy': 0.8192079067230225,
'specificity': 0.9046558737754822,
'miss_rate': 0.26624006032943726,
'fall_out': 0.09534412622451782,
'mcc': 0.6368142366409302}}],
'tuning_False': [{'train_evaluation': {'loss': 0.9460542798042297,
'accuracy': 0.870945155620575,
'recall': 0.8147878646850586,
'precision': 0.7213374972343445,
'AUROC': 0.9330935478210449,
'AUPRC': 0.7831394076347351,
'f1_score': 0.7652201652526855,
'balanced_accuracy': 0.85263592004776,
'specificity': 0.8904839754104614,
'miss_rate': 0.1852121204137802,
'fall_out': 0.10951601713895798,
'mcc': 0.6790767908096313},
'test_evaluation': {'loss': 0.9798312783241272,
'accuracy': 0.849727213382721,
'recall': 0.7727360725402832,
'precision': 0.6852966547012329,
'AUROC': 0.9128032326698303,
'AUPRC': 0.7442159652709961,
'f1_score': 0.7263944745063782,
'balanced_accuracy': 0.824627161026001,
'specificity': 0.876518189907074,
'miss_rate': 0.2272639125585556,
'fall_out': 0.12348178029060364,
'mcc': 0.6254611611366272}},
{'train_evaluation': {'loss': 1.1601184606552124,
'accuracy': 0.8567405939102173,
'recall': 0.7512242197990417,
'precision': 0.710408091545105,
'AUROC': 0.921216607093811,
'AUPRC': 0.7549236416816711,
'f1_score': 0.7302462458610535,
'balanced_accuracy': 0.8223385810852051,
'specificity': 0.8934530019760132,
'miss_rate': 0.24877575039863586,
'fall_out': 0.10654700547456741,
'mcc': 0.6332806944847107},
'test_evaluation': {'loss': 1.1724567413330078,
'accuracy': 0.8513790965080261,
'recall': 0.7372503280639648,
'precision': 0.7019941210746765,
'AUROC': 0.912536084651947,
'AUPRC': 0.7344838380813599,
'f1_score': 0.7191904187202454,
'balanced_accuracy': 0.8141717314720154,
'specificity': 0.8910931348800659,
'miss_rate': 0.26274967193603516,
'fall_out': 0.10890688002109528,
'mcc': 0.618564248085022}},
{'train_evaluation': {'loss': 1.179643154144287,
'accuracy': 0.8540248274803162,
'recall': 0.7380363345146179,
'precision': 0.7085602283477783,
'AUROC': 0.9189560413360596,
'AUPRC': 0.7481982111930847,
'f1_score': 0.7229980230331421,
'balanced_accuracy': 0.8162086009979248,
'specificity': 0.8943808078765869,
'miss_rate': 0.2619636356830597,
'fall_out': 0.10561919212341309,
'mcc': 0.6241933703422546},
'test_evaluation': {'loss': 1.1890933513641357,
'accuracy': 0.8505781888961792,
'recall': 0.7362807989120483,
'precision': 0.700295090675354,
'AUROC': 0.9114962816238403,
'AUPRC': 0.7251158952713013,
'f1_score': 0.7178372144699097,
'balanced_accuracy': 0.8133158683776855,
'specificity': 0.890350878238678,
'miss_rate': 0.26371923089027405,
'fall_out': 0.10964912176132202,
'mcc': 0.6166647672653198}},
{'train_evaluation': {'loss': 1.1561720371246338,
'accuracy': 0.858555257320404,
'recall': 0.7699878811836243,
'precision': 0.7077409625053406,
'AUROC': 0.9223573803901672,
'AUPRC': 0.7560591697692871,
'f1_score': 0.7375534176826477,
'balanced_accuracy': 0.8296792507171631,
'specificity': 0.8893706202507019,
'miss_rate': 0.23001211881637573,
'fall_out': 0.11062939465045929,
'mcc': 0.6420451402664185},
'test_evaluation': {'loss': 1.166853666305542,
'accuracy': 0.8519297242164612,
'recall': 0.7552840709686279,
'precision': 0.6966553330421448,
'AUROC': 0.9151421189308167,
'AUPRC': 0.739275336265564,
'f1_score': 0.7247859835624695,
'balanced_accuracy': 0.8204220533370972,
'specificity': 0.8855600357055664,
'miss_rate': 0.24471591413021088,
'fall_out': 0.1144399493932724,
'mcc': 0.6246839165687561}},
{'train_evaluation': {'loss': 1.1150840520858765,
'accuracy': 0.8573788404464722,
'recall': 0.770763635635376,
'precision': 0.7044981122016907,
'AUROC': 0.9207304120063782,
'AUPRC': 0.7530894875526428,
'f1_score': 0.736142635345459,
'balanced_accuracy': 0.8291392922401428,
'specificity': 0.8875149488449097,
'miss_rate': 0.22923636436462402,
'fall_out': 0.11248502880334854,
'mcc': 0.6399033665657043},
'test_evaluation': {'loss': 1.1292587518692017,
'accuracy': 0.8479751944541931,
'recall': 0.7465580701828003,
'precision': 0.6899641752243042,
'AUROC': 0.9115030169487,
'AUPRC': 0.7361984252929688,
'f1_score': 0.7171463370323181,
'balanced_accuracy': 0.814911961555481,
'specificity': 0.8832658529281616,
'miss_rate': 0.2534419298171997,
'fall_out': 0.11673413962125778,
'mcc': 0.6143116354942322}},
{'train_evaluation': {'loss': 1.1825459003448486,
'accuracy': 0.8627603054046631,
'recall': 0.7865697145462036,
'precision': 0.7119410037994385,
'AUROC': 0.9254060387611389,
'AUPRC': 0.763757586479187,
'f1_score': 0.7473970055580139,
'balanced_accuracy': 0.8379195928573608,
'specificity': 0.8892694115638733,
'miss_rate': 0.21343030035495758,
'fall_out': 0.1107306107878685,
'mcc': 0.6550349593162537},
'test_evaluation': {'loss': 1.2020601034164429,
'accuracy': 0.849727213382721,
'recall': 0.7707969546318054,
'precision': 0.6859361529350281,
'AUROC': 0.9125322699546814,
'AUPRC': 0.73936927318573,
'f1_score': 0.7258948087692261,
'balanced_accuracy': 0.8239949941635132,
'specificity': 0.8771929740905762,
'miss_rate': 0.2292030304670334,
'fall_out': 0.12280701845884323,
'mcc': 0.6248783469200134}},
{'train_evaluation': {'loss': 1.1051498651504517,
'accuracy': 0.8636614084243774,
'recall': 0.8064484596252441,
'precision': 0.7067346572875977,
'AUROC': 0.9259819984436035,
'AUPRC': 0.76543790102005,
'f1_score': 0.7533061504364014,
'balanced_accuracy': 0.8450080156326294,
'specificity': 0.8835675120353699,
'miss_rate': 0.19355151057243347,
'fall_out': 0.11643246561288834,
'mcc': 0.6624137163162231},
'test_evaluation': {'loss': 1.1274179220199585,
'accuracy': 0.8476748466491699,
'recall': 0.7793290615081787,
'precision': 0.6784267425537109,
'AUROC': 0.9110085964202881,
'AUPRC': 0.7317487597465515,
'f1_score': 0.7253857851028442,
'balanced_accuracy': 0.8253933191299438,
'specificity': 0.8714575171470642,
'miss_rate': 0.2206709384918213,
'fall_out': 0.12854251265525818,
'mcc': 0.6235470175743103}},
{'train_evaluation': {'loss': 1.1710618734359741,
'accuracy': 0.8559771776199341,
'recall': 0.7828848361968994,
'precision': 0.6966820359230042,
'AUROC': 0.9197404384613037,
'AUPRC': 0.7510890364646912,
'f1_score': 0.7372722625732422,
'balanced_accuracy': 0.8321465253829956,
'specificity': 0.8814082741737366,
'miss_rate': 0.2171151489019394,
'fall_out': 0.11859174072742462,
'mcc': 0.6405951380729675},
'test_evaluation': {'loss': 1.182309865951538,
'accuracy': 0.8485758900642395,
'recall': 0.7642039656639099,
'precision': 0.6853913068771362,
'AUROC': 0.9110748171806335,
'AUPRC': 0.7294859290122986,
'f1_score': 0.722655177116394,
'balanced_accuracy': 0.821069598197937,
'specificity': 0.8779352307319641,
'miss_rate': 0.2357960045337677,
'fall_out': 0.12206477671861649,
'mcc': 0.6206707954406738}},
{'train_evaluation': {'loss': 1.1658203601837158,
'accuracy': 0.8530486822128296,
'recall': 0.7403151392936707,
'precision': 0.7051027417182922,
'AUROC': 0.9172919988632202,
'AUPRC': 0.7413213849067688,
'f1_score': 0.7222800254821777,
'balanced_accuracy': 0.8162935972213745,
'specificity': 0.8922721147537231,
'miss_rate': 0.25968486070632935,
'fall_out': 0.10772786289453506,
'mcc': 0.6227942109107971},
'test_evaluation': {'loss': 1.1778863668441772,
'accuracy': 0.8484256863594055,
'recall': 0.736668586730957,
'precision': 0.6946425437927246,
'AUROC': 0.9096946716308594,
'AUPRC': 0.7292711734771729,
'f1_score': 0.7150385975837708,
'balanced_accuracy': 0.8119915127754211,
'specificity': 0.8873144388198853,
'miss_rate': 0.2633313834667206,
'fall_out': 0.11268556118011475,
'mcc': 0.6124035716056824}},
{'train_evaluation': {'loss': 1.1406265497207642,
'accuracy': 0.858167290687561,
'recall': 0.7710545659065247,
'precision': 0.7063604593276978,
'AUROC': 0.9218806028366089,
'AUPRC': 0.7564809918403625,
'f1_score': 0.7372910976409912,
'balanced_accuracy': 0.8297655582427979,
'specificity': 0.888476550579071,
'miss_rate': 0.22894544899463654,
'fall_out': 0.11152347177267075,
'mcc': 0.6415613889694214},
'test_evaluation': {'loss': 1.1517939567565918,
'accuracy': 0.8526805639266968,
'recall': 0.7552840709686279,
'precision': 0.6985294222831726,
'AUROC': 0.9148688316345215,
'AUPRC': 0.7383679151535034,
'f1_score': 0.7257989645004272,
'balanced_accuracy': 0.8209280967712402,
'specificity': 0.8865721821784973,
'miss_rate': 0.24471591413021088,
'fall_out': 0.11342780292034149,
'mcc': 0.6261851787567139}}]}
# Saving results
import json
# Saving metrics
file = open("results/MMNN_metrics_promoters.txt", "w")
string = repr(MMNN_metrics_promoters)
file.write("MMNN_metrics_promoters = " + string + "\n")
file.close()
# Saving history
for tuning in ("tuning_True", "tuning_False"):
with open("results/MMNN_history_promoters_" + tuning + ".json", 'w') as h:
for holdout in range(number_of_splits):
json.dump(MMNN_history_promoters[tuning][holdout].history, h)
MMNN_metrics_promoters
{'tuning_True': [{'train_evaluation': {'loss': 0.26318269968032837,
'accuracy': 0.8808194994926453,
'recall': 0.81207275390625,
'precision': 0.7478567361831665,
'AUROC': 0.9440953135490417,
'AUPRC': 0.8309550881385803,
'f1_score': 0.778643012046814,
'balanced_accuracy': 0.8584057092666626,
'specificity': 0.9047386050224304,
'miss_rate': 0.1879272758960724,
'fall_out': 0.09526138752698898,
'mcc': 0.6983979344367981},
'test_evaluation': {'loss': 0.32070329785346985,
'accuracy': 0.8511788845062256,
'recall': 0.7537328004837036,
'precision': 0.695348858833313,
'AUROC': 0.9124017953872681,
'AUPRC': 0.7418513894081116,
'f1_score': 0.7233646512031555,
'balanced_accuracy': 0.8194102644920349,
'specificity': 0.8850877285003662,
'miss_rate': 0.24626721441745758,
'fall_out': 0.11491227895021439,
'mcc': 0.6227453947067261}},
{'train_evaluation': {'loss': 0.2676412761211395,
'accuracy': 0.8855876922607422,
'recall': 0.7583515048027039,
'precision': 0.7899894118309021,
'AUROC': 0.9453145265579224,
'AUPRC': 0.8497247695922852,
'f1_score': 0.773847222328186,
'balanced_accuracy': 0.8441042900085449,
'specificity': 0.9298571348190308,
'miss_rate': 0.24164848029613495,
'fall_out': 0.07014288008213043,
'mcc': 0.6975745558738708},
'test_evaluation': {'loss': 0.32417988777160645,
'accuracy': 0.84877610206604,
'recall': 0.6812099814414978,
'precision': 0.7184048891067505,
'AUROC': 0.9119037389755249,
'AUPRC': 0.7427470088005066,
'f1_score': 0.699313223361969,
'balanced_accuracy': 0.7941474914550781,
'specificity': 0.9070850014686584,
'miss_rate': 0.3187899887561798,
'fall_out': 0.09291497617959976,
'mcc': 0.5987727046012878}},
{'train_evaluation': {'loss': 0.278840035200119,
'accuracy': 0.8717461228370667,
'recall': 0.8497454524040222,
'precision': 0.7102735638618469,
'AUROC': 0.9386857748031616,
'AUPRC': 0.8237740397453308,
'f1_score': 0.7737748622894287,
'balanced_accuracy': 0.8645731210708618,
'specificity': 0.8794007897377014,
'miss_rate': 0.15025454759597778,
'fall_out': 0.12059920281171799,
'mcc': 0.6906370520591736},
'test_evaluation': {'loss': 0.3144226670265198,
'accuracy': 0.8533313274383545,
'recall': 0.8206321597099304,
'precision': 0.6785313487052917,
'AUROC': 0.9179574847221375,
'AUPRC': 0.7553421854972839,
'f1_score': 0.7428470849990845,
'balanced_accuracy': 0.8426710367202759,
'specificity': 0.8647098541259766,
'miss_rate': 0.17936785519123077,
'fall_out': 0.13529014587402344,
'mcc': 0.6472153067588806}},
{'train_evaluation': {'loss': 0.27988481521606445,
'accuracy': 0.8793051838874817,
'recall': 0.6906181573867798,
'precision': 0.8136174082756042,
'AUROC': 0.940905749797821,
'AUPRC': 0.8470730185508728,
'f1_score': 0.7470890879631042,
'balanced_accuracy': 0.817786693572998,
'specificity': 0.9449552297592163,
'miss_rate': 0.3093818128108978,
'fall_out': 0.055044788867235184,
'mcc': 0.6723964214324951},
'test_evaluation': {'loss': 0.3167767822742462,
'accuracy': 0.8466235995292664,
'recall': 0.6218731999397278,
'precision': 0.7421892881393433,
'AUROC': 0.9177714586257935,
'AUPRC': 0.7548065185546875,
'f1_score': 0.6767250299453735,
'balanced_accuracy': 0.7733522653579712,
'specificity': 0.9248313307762146,
'miss_rate': 0.3781268298625946,
'fall_out': 0.07516869157552719,
'mcc': 0.581089198589325}},
{'train_evaluation': {'loss': 0.2916513979434967,
'accuracy': 0.8706072568893433,
'recall': 0.7613090872764587,
'precision': 0.7435362935066223,
'AUROC': 0.9325039386749268,
'AUPRC': 0.8047640919685364,
'f1_score': 0.7523177266120911,
'balanced_accuracy': 0.8349722623825073,
'specificity': 0.9086354374885559,
'miss_rate': 0.23869091272354126,
'fall_out': 0.09136456251144409,
'mcc': 0.664848804473877},
'test_evaluation': {'loss': 0.32179221510887146,
'accuracy': 0.8513290286064148,
'recall': 0.7196044325828552,
'precision': 0.7088825106620789,
'AUROC': 0.912771999835968,
'AUPRC': 0.7403241991996765,
'f1_score': 0.7142032384872437,
'balanced_accuracy': 0.8083852529525757,
'specificity': 0.8971660137176514,
'miss_rate': 0.2803955674171448,
'fall_out': 0.10283400863409042,
'mcc': 0.6137756109237671}},
{'train_evaluation': {'loss': 0.2455703467130661,
'accuracy': 0.8958625197410583,
'recall': 0.7903030514717102,
'precision': 0.8031139373779297,
'AUROC': 0.9553269743919373,
'AUPRC': 0.8772685527801514,
'f1_score': 0.7966569662094116,
'balanced_accuracy': 0.8614464998245239,
'specificity': 0.9325899481773376,
'miss_rate': 0.209696963429451,
'fall_out': 0.06741004437208176,
'mcc': 0.7267146706581116},
'test_evaluation': {'loss': 0.3089958727359772,
'accuracy': 0.8547829985618591,
'recall': 0.7132053375244141,
'precision': 0.7211764454841614,
'AUROC': 0.9196145534515381,
'AUPRC': 0.7678280472755432,
'f1_score': 0.7171687483787537,
'balanced_accuracy': 0.8086269497871399,
'specificity': 0.9040485620498657,
'miss_rate': 0.28679466247558594,
'fall_out': 0.09595141559839249,
'mcc': 0.6195034980773926}},
{'train_evaluation': {'loss': 0.2713947594165802,
'accuracy': 0.8753003478050232,
'recall': 0.8276363611221313,
'precision': 0.7270326614379883,
'AUROC': 0.9408495426177979,
'AUPRC': 0.8237742781639099,
'f1_score': 0.774079442024231,
'balanced_accuracy': 0.8597602844238281,
'specificity': 0.8918841481208801,
'miss_rate': 0.17236363887786865,
'fall_out': 0.10811585932970047,
'mcc': 0.6912178993225098},
'test_evaluation': {'loss': 0.32190337777137756,
'accuracy': 0.8479751944541931,
'recall': 0.7727360725402832,
'precision': 0.6811965703964233,
'AUROC': 0.9124097228050232,
'AUPRC': 0.7414856553077698,
'f1_score': 0.7240846753120422,
'balanced_accuracy': 0.8234462738037109,
'specificity': 0.8741565346717834,
'miss_rate': 0.2272639125585556,
'fall_out': 0.12584345042705536,
'mcc': 0.6220882534980774}},
{'train_evaluation': {'loss': 0.3518398404121399,
'accuracy': 0.8440253138542175,
'recall': 0.8076121211051941,
'precision': 0.6622534990310669,
'AUROC': 0.9103205800056458,
'AUPRC': 0.7291437387466431,
'f1_score': 0.7277453541755676,
'balanced_accuracy': 0.8321533799171448,
'specificity': 0.8566946387290955,
'miss_rate': 0.1923878788948059,
'fall_out': 0.14330539107322693,
'mcc': 0.6259358525276184},
'test_evaluation': {'loss': 0.35547134280204773,
'accuracy': 0.8427191376686096,
'recall': 0.8006592988967896,
'precision': 0.6613807678222656,
'AUROC': 0.9051212072372437,
'AUPRC': 0.7107263803482056,
'f1_score': 0.7243859767913818,
'balanced_accuracy': 0.8290071487426758,
'specificity': 0.8573549389839172,
'miss_rate': 0.19934070110321045,
'fall_out': 0.14264507591724396,
'mcc': 0.6212447881698608}},
{'train_evaluation': {'loss': 0.22503067553043365,
'accuracy': 0.9078769683837891,
'recall': 0.7489939332008362,
'precision': 0.8761342763900757,
'AUROC': 0.9635559320449829,
'AUPRC': 0.9075829386711121,
'f1_score': 0.8075907826423645,
'balanced_accuracy': 0.856075644493103,
'specificity': 0.9631572961807251,
'miss_rate': 0.2510060667991638,
'fall_out': 0.03684272617101669,
'mcc': 0.7514887452125549},
'test_evaluation': {'loss': 0.32369884848594666,
'accuracy': 0.8450718522071838,
'recall': 0.6290478706359863,
'precision': 0.7329416871070862,
'AUROC': 0.9145113825798035,
'AUPRC': 0.7565793395042419,
'f1_score': 0.6770322322845459,
'balanced_accuracy': 0.7746453881263733,
'specificity': 0.9202429056167603,
'miss_rate': 0.3709520995616913,
'fall_out': 0.07975708693265915,
'mcc': 0.5788154602050781}},
{'train_evaluation': {'loss': 0.23042787611484528,
'accuracy': 0.903446614742279,
'recall': 0.8192969560623169,
'precision': 0.8090587258338928,
'AUROC': 0.9599372148513794,
'AUPRC': 0.8916779160499573,
'f1_score': 0.8141456246376038,
'balanced_accuracy': 0.8760108947753906,
'specificity': 0.9327248930931091,
'miss_rate': 0.1807030290365219,
'fall_out': 0.06727509200572968,
'mcc': 0.748958945274353},
'test_evaluation': {'loss': 0.30608025193214417,
'accuracy': 0.8605396151542664,
'recall': 0.7337599396705627,
'precision': 0.7281123995780945,
'AUROC': 0.9219881892204285,
'AUPRC': 0.7736676335334778,
'f1_score': 0.7309252619743347,
'balanced_accuracy': 0.8192079067230225,
'specificity': 0.9046558737754822,
'miss_rate': 0.26624006032943726,
'fall_out': 0.09534412622451782,
'mcc': 0.6368142366409302}}],
'tuning_False': [{'train_evaluation': {'loss': 0.9460542798042297,
'accuracy': 0.870945155620575,
'recall': 0.8147878646850586,
'precision': 0.7213374972343445,
'AUROC': 0.9330935478210449,
'AUPRC': 0.7831394076347351,
'f1_score': 0.7652201652526855,
'balanced_accuracy': 0.85263592004776,
'specificity': 0.8904839754104614,
'miss_rate': 0.1852121204137802,
'fall_out': 0.10951601713895798,
'mcc': 0.6790767908096313},
'test_evaluation': {'loss': 0.9798312783241272,
'accuracy': 0.849727213382721,
'recall': 0.7727360725402832,
'precision': 0.6852966547012329,
'AUROC': 0.9128032326698303,
'AUPRC': 0.7442159652709961,
'f1_score': 0.7263944745063782,
'balanced_accuracy': 0.824627161026001,
'specificity': 0.876518189907074,
'miss_rate': 0.2272639125585556,
'fall_out': 0.12348178029060364,
'mcc': 0.6254611611366272}},
{'train_evaluation': {'loss': 1.1601184606552124,
'accuracy': 0.8567405939102173,
'recall': 0.7512242197990417,
'precision': 0.710408091545105,
'AUROC': 0.921216607093811,
'AUPRC': 0.7549236416816711,
'f1_score': 0.7302462458610535,
'balanced_accuracy': 0.8223385810852051,
'specificity': 0.8934530019760132,
'miss_rate': 0.24877575039863586,
'fall_out': 0.10654700547456741,
'mcc': 0.6332806944847107},
'test_evaluation': {'loss': 1.1724567413330078,
'accuracy': 0.8513790965080261,
'recall': 0.7372503280639648,
'precision': 0.7019941210746765,
'AUROC': 0.912536084651947,
'AUPRC': 0.7344838380813599,
'f1_score': 0.7191904187202454,
'balanced_accuracy': 0.8141717314720154,
'specificity': 0.8910931348800659,
'miss_rate': 0.26274967193603516,
'fall_out': 0.10890688002109528,
'mcc': 0.618564248085022}},
{'train_evaluation': {'loss': 1.179643154144287,
'accuracy': 0.8540248274803162,
'recall': 0.7380363345146179,
'precision': 0.7085602283477783,
'AUROC': 0.9189560413360596,
'AUPRC': 0.7481982111930847,
'f1_score': 0.7229980230331421,
'balanced_accuracy': 0.8162086009979248,
'specificity': 0.8943808078765869,
'miss_rate': 0.2619636356830597,
'fall_out': 0.10561919212341309,
'mcc': 0.6241933703422546},
'test_evaluation': {'loss': 1.1890933513641357,
'accuracy': 0.8505781888961792,
'recall': 0.7362807989120483,
'precision': 0.700295090675354,
'AUROC': 0.9114962816238403,
'AUPRC': 0.7251158952713013,
'f1_score': 0.7178372144699097,
'balanced_accuracy': 0.8133158683776855,
'specificity': 0.890350878238678,
'miss_rate': 0.26371923089027405,
'fall_out': 0.10964912176132202,
'mcc': 0.6166647672653198}},
{'train_evaluation': {'loss': 1.1561720371246338,
'accuracy': 0.858555257320404,
'recall': 0.7699878811836243,
'precision': 0.7077409625053406,
'AUROC': 0.9223573803901672,
'AUPRC': 0.7560591697692871,
'f1_score': 0.7375534176826477,
'balanced_accuracy': 0.8296792507171631,
'specificity': 0.8893706202507019,
'miss_rate': 0.23001211881637573,
'fall_out': 0.11062939465045929,
'mcc': 0.6420451402664185},
'test_evaluation': {'loss': 1.166853666305542,
'accuracy': 0.8519297242164612,
'recall': 0.7552840709686279,
'precision': 0.6966553330421448,
'AUROC': 0.9151421189308167,
'AUPRC': 0.739275336265564,
'f1_score': 0.7247859835624695,
'balanced_accuracy': 0.8204220533370972,
'specificity': 0.8855600357055664,
'miss_rate': 0.24471591413021088,
'fall_out': 0.1144399493932724,
'mcc': 0.6246839165687561}},
{'train_evaluation': {'loss': 1.1150840520858765,
'accuracy': 0.8573788404464722,
'recall': 0.770763635635376,
'precision': 0.7044981122016907,
'AUROC': 0.9207304120063782,
'AUPRC': 0.7530894875526428,
'f1_score': 0.736142635345459,
'balanced_accuracy': 0.8291392922401428,
'specificity': 0.8875149488449097,
'miss_rate': 0.22923636436462402,
'fall_out': 0.11248502880334854,
'mcc': 0.6399033665657043},
'test_evaluation': {'loss': 1.1292587518692017,
'accuracy': 0.8479751944541931,
'recall': 0.7465580701828003,
'precision': 0.6899641752243042,
'AUROC': 0.9115030169487,
'AUPRC': 0.7361984252929688,
'f1_score': 0.7171463370323181,
'balanced_accuracy': 0.814911961555481,
'specificity': 0.8832658529281616,
'miss_rate': 0.2534419298171997,
'fall_out': 0.11673413962125778,
'mcc': 0.6143116354942322}},
{'train_evaluation': {'loss': 1.1825459003448486,
'accuracy': 0.8627603054046631,
'recall': 0.7865697145462036,
'precision': 0.7119410037994385,
'AUROC': 0.9254060387611389,
'AUPRC': 0.763757586479187,
'f1_score': 0.7473970055580139,
'balanced_accuracy': 0.8379195928573608,
'specificity': 0.8892694115638733,
'miss_rate': 0.21343030035495758,
'fall_out': 0.1107306107878685,
'mcc': 0.6550349593162537},
'test_evaluation': {'loss': 1.2020601034164429,
'accuracy': 0.849727213382721,
'recall': 0.7707969546318054,
'precision': 0.6859361529350281,
'AUROC': 0.9125322699546814,
'AUPRC': 0.73936927318573,
'f1_score': 0.7258948087692261,
'balanced_accuracy': 0.8239949941635132,
'specificity': 0.8771929740905762,
'miss_rate': 0.2292030304670334,
'fall_out': 0.12280701845884323,
'mcc': 0.6248783469200134}},
{'train_evaluation': {'loss': 1.1051498651504517,
'accuracy': 0.8636614084243774,
'recall': 0.8064484596252441,
'precision': 0.7067346572875977,
'AUROC': 0.9259819984436035,
'AUPRC': 0.76543790102005,
'f1_score': 0.7533061504364014,
'balanced_accuracy': 0.8450080156326294,
'specificity': 0.8835675120353699,
'miss_rate': 0.19355151057243347,
'fall_out': 0.11643246561288834,
'mcc': 0.6624137163162231},
'test_evaluation': {'loss': 1.1274179220199585,
'accuracy': 0.8476748466491699,
'recall': 0.7793290615081787,
'precision': 0.6784267425537109,
'AUROC': 0.9110085964202881,
'AUPRC': 0.7317487597465515,
'f1_score': 0.7253857851028442,
'balanced_accuracy': 0.8253933191299438,
'specificity': 0.8714575171470642,
'miss_rate': 0.2206709384918213,
'fall_out': 0.12854251265525818,
'mcc': 0.6235470175743103}},
{'train_evaluation': {'loss': 1.1710618734359741,
'accuracy': 0.8559771776199341,
'recall': 0.7828848361968994,
'precision': 0.6966820359230042,
'AUROC': 0.9197404384613037,
'AUPRC': 0.7510890364646912,
'f1_score': 0.7372722625732422,
'balanced_accuracy': 0.8321465253829956,
'specificity': 0.8814082741737366,
'miss_rate': 0.2171151489019394,
'fall_out': 0.11859174072742462,
'mcc': 0.6405951380729675},
'test_evaluation': {'loss': 1.182309865951538,
'accuracy': 0.8485758900642395,
'recall': 0.7642039656639099,
'precision': 0.6853913068771362,
'AUROC': 0.9110748171806335,
'AUPRC': 0.7294859290122986,
'f1_score': 0.722655177116394,
'balanced_accuracy': 0.821069598197937,
'specificity': 0.8779352307319641,
'miss_rate': 0.2357960045337677,
'fall_out': 0.12206477671861649,
'mcc': 0.6206707954406738}},
{'train_evaluation': {'loss': 1.1658203601837158,
'accuracy': 0.8530486822128296,
'recall': 0.7403151392936707,
'precision': 0.7051027417182922,
'AUROC': 0.9172919988632202,
'AUPRC': 0.7413213849067688,
'f1_score': 0.7222800254821777,
'balanced_accuracy': 0.8162935972213745,
'specificity': 0.8922721147537231,
'miss_rate': 0.25968486070632935,
'fall_out': 0.10772786289453506,
'mcc': 0.6227942109107971},
'test_evaluation': {'loss': 1.1778863668441772,
'accuracy': 0.8484256863594055,
'recall': 0.736668586730957,
'precision': 0.6946425437927246,
'AUROC': 0.9096946716308594,
'AUPRC': 0.7292711734771729,
'f1_score': 0.7150385975837708,
'balanced_accuracy': 0.8119915127754211,
'specificity': 0.8873144388198853,
'miss_rate': 0.2633313834667206,
'fall_out': 0.11268556118011475,
'mcc': 0.6124035716056824}},
{'train_evaluation': {'loss': 1.1406265497207642,
'accuracy': 0.858167290687561,
'recall': 0.7710545659065247,
'precision': 0.7063604593276978,
'AUROC': 0.9218806028366089,
'AUPRC': 0.7564809918403625,
'f1_score': 0.7372910976409912,
'balanced_accuracy': 0.8297655582427979,
'specificity': 0.888476550579071,
'miss_rate': 0.22894544899463654,
'fall_out': 0.11152347177267075,
'mcc': 0.6415613889694214},
'test_evaluation': {'loss': 1.1517939567565918,
'accuracy': 0.8526805639266968,
'recall': 0.7552840709686279,
'precision': 0.6985294222831726,
'AUROC': 0.9148688316345215,
'AUPRC': 0.7383679151535034,
'f1_score': 0.7257989645004272,
'balanced_accuracy': 0.8209280967712402,
'specificity': 0.8865721821784973,
'miss_rate': 0.24471591413021088,
'fall_out': 0.11342780292034149,
'mcc': 0.6261851787567139}}]}
MMNN_distinct_metrics_promoters = {}
MMNN_distinct_metrics_promoters["tuning_True"] = {}
MMNN_distinct_metrics_promoters["tuning_False"] = {}
for tuning in ("tuning_True", "tuning_False"):
MMNN_distinct_metrics_promoters[tuning]["accuracy_test"] = []
MMNN_distinct_metrics_promoters[tuning]["AUROC_test"] = []
MMNN_distinct_metrics_promoters[tuning]["AUPRC_test"] = []
for holdout in MMNN_metrics_promoters[tuning]:
MMNN_distinct_metrics_promoters[tuning]["accuracy_test"].append(holdout["test_evaluation"]["accuracy"])
MMNN_distinct_metrics_promoters[tuning]["AUROC_test"].append(holdout["test_evaluation"]["AUROC"])
MMNN_distinct_metrics_promoters[tuning]["AUPRC_test"].append(holdout["test_evaluation"]["AUPRC"])
print("MMNN - Promoters - Wilcoxon test - fixed MLP and CNN performances VS optimized MLP and CNN performances")
print(f"-Accuracy: {wilcoxon(MMNN_distinct_metrics_promoters['tuning_True']['accuracy_test'], MMNN_distinct_metrics_promoters['tuning_False']['accuracy_test'])}")
print(f"-AUROC: {wilcoxon(MMNN_distinct_metrics_promoters['tuning_True']['AUROC_test'], MMNN_distinct_metrics_promoters['tuning_False']['AUROC_test'])}")
print(f"-AUPRC: {wilcoxon(MMNN_distinct_metrics_promoters['tuning_True']['AUPRC_test'], MMNN_distinct_metrics_promoters['tuning_False']['AUPRC_test'])}")
MMNN - Promoters - Wilcoxon test - fixed MLP and CNN performances VS optimized MLP and CNN performances -Accuracy: WilcoxonResult(statistic=25.5, pvalue=0.921875) -AUROC: WilcoxonResult(statistic=10.0, pvalue=0.083984375) -AUPRC: WilcoxonResult(statistic=7.0, pvalue=0.037109375)
print("---- ENHANCERS ----")
MMNN_metrics_enhancers = {}
MMNN_history_enhancers = {}
MMNN_metrics_enhancers["tuning_True"] = []
MMNN_metrics_enhancers["tuning_False"] = []
MMNN_history_enhancers["tuning_True"] = []
MMNN_history_enhancers["tuning_False"] = []
for holdout_number, (train_indices, test_indices) in enumerate(holdouts_generator.split(bed_enhancers, enhancers_labels)):
print(f"-- HOLDOUT {holdout_number+1} --")
# Train/Test data
x_train_enhancers, x_test_enhancers = enhancers_epigenomes.iloc[train_indices], enhancers_epigenomes.iloc[test_indices]
x_train_enhancers_bed, x_test_enhancers_bed = bed_enhancers.iloc[train_indices], bed_enhancers.iloc[test_indices]
y_train_enhancers, y_test_enhancers = enhancers_labels.iloc[train_indices], enhancers_labels.iloc[test_indices]
correlation_test = False
for use_tuning in (False, True):
if use_tuning:
## Hyperparameter tuning
# Generate holdouts
for holdout_number_tuning, (train_indices_tuning, val_indices_tuning) in enumerate(holdouts_generator_tuning.split(x_train_enhancers, y_train_enhancers)):
# Train/Validation data
x_train_enhancers_tuning, x_val_enhancers_tuning = x_train_enhancers.iloc[train_indices_tuning], x_train_enhancers.iloc[val_indices_tuning]
x_train_enhancers_bed_tuning, x_val_enhancers_bed_tuning = x_train_enhancers_bed.iloc[train_indices_tuning], x_train_enhancers_bed.iloc[val_indices_tuning]
y_train_enhancers_tuning, y_val_enhancers_tuning = y_train_enhancers.iloc[train_indices_tuning], y_train_enhancers.iloc[val_indices_tuning]
hp = kt.HyperParameters()
best_hyperparameters_MLP = hyperparameter_tuning(
x_train_enhancers_tuning.values,
x_val_enhancers_tuning.values,
y_train_enhancers_tuning.values,
y_val_enhancers_tuning.values,
build_MLP_hypermodel,
name = "MLP_hypermodel_enhancers_" + str(holdout_number),
directory='MLP_hypermodel_enhancers',
max_trials = 8,
epochs = 50,
batch_size = 256
)
best_hyperparameters_CNN = hyperparameter_tuning_CNN(
x_train_enhancers_bed_tuning,
x_val_enhancers_bed_tuning,
y_train_enhancers_tuning.values,
y_val_enhancers_tuning.values,
build_CNN_hypermodel,
name = "CNN_hypermodel_enhancers_" + str(holdout_number),
directory='CNN_hypermodel_enhancers',
max_trials = 15,
epochs = 50,
batch_size = 256
)
if not correlation_test:
useless_features = []
## Remove uncorrelated features with the output (epigenomic data)
uncorrelated_features = uncorrelated_features_test(x_train_enhancers, y_train_enhancers)
for feature in (x_train_enhancers.columns):
if feature in (uncorrelated_features):
x_train_enhancers_subset = x_train_enhancers.drop(columns=feature)
x_test_enhancers_subset = x_test_enhancers.drop(columns=feature)
useless_features.append(feature)
## Remove correlated features with eachother (epigenomic data)
correlated_features = correlated_features_test(x_train_enhancers_subset)
for feature in (x_train_enhancers_subset.columns):
if feature in (correlated_features):
x_train_enhancers_subset = x_train_enhancers_subset.drop(columns=feature)
x_test_enhancers_subset = x_test_enhancers_subset.drop(columns=feature)
useless_features.append(feature)
if correlation_test:
# Correlation tests already executed, removing useless features found
print(f"- Removing uncorrelated/correlated features: {useless_features}\n")
for feature in (x_train_enhancers.columns):
if feature in (useless_features):
x_train_enhancers_subset = x_train_enhancers.drop(columns=feature)
x_test_enhancers_subset = x_test_enhancers.drop(columns=feature)
correlation_test = True
if not use_tuning:
# Build Fixed MLP, CNN
MLP, input_epigenomic_data, last_hidden_MLP = build_fixed_MLP(x_train_enhancers_subset.shape)
CNN, input_sequence_data, last_hidden_CNN = build_fixed_CNN(window_size)
if use_tuning:
# Build MLP, CNN with best set of hyperparameters
MLP, input_epigenomic_data, last_hidden_MLP = build_optimized_MLP(best_hyperparameters_MLP, x_train_enhancers_subset.shape)
CNN, input_sequence_data, last_hidden_CNN = build_optimized_CNN(best_hyperparameters_CNN, window_size)
# Build MMNN
MMNN = build_MMNN(
input_epigenomic_data,
input_sequence_data,
last_hidden_MLP,
last_hidden_CNN
)
print("- Training MMNN model:\n")
MMNN_holdout_metrics, MMNN_holdout_history = train_MMNN(
MMNN,
x_train_enhancers_bed,
x_test_enhancers_bed,
x_train_enhancers_subset.values,
x_test_enhancers_subset.values,
y_train_enhancers.values,
y_test_enhancers.values,
genome,
epochs,
batch_size
)
MMNN_metrics_enhancers["tuning_" + str(use_tuning)].append(MMNN_holdout_metrics)
MMNN_history_enhancers["tuning_" + str(use_tuning)].append(MMNN_holdout_history)
---- ENHANCERS ----
-- HOLDOUT 1 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['SNRNP70', 'ZNF282']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70', 'ZNF282']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 18s 68ms/step - loss: 17.3500 - accuracy: 0.8954 - recall: 0.2342 - precision: 0.5988 - AUROC: 0.7393 - AUPRC: 0.3820 - f1_score: 0.3368 - balanced_accuracy: 0.6071 - specificity: 0.9799 - miss_rate: 0.7658 - fall_out: 0.0201 - mcc: 0.3298 - val_loss: 2.6032 - val_accuracy: 0.9010 - val_recall: 0.1429 - val_precision: 0.8991 - val_AUROC: 0.8457 - val_AUPRC: 0.5712 - val_f1_score: 0.2465 - val_balanced_accuracy: 0.5704 - val_specificity: 0.9980 - val_miss_rate: 0.8571 - val_fall_out: 0.0020 - val_mcc: 0.3357
Epoch 2/1000
198/198 [==============================] - 13s 60ms/step - loss: 2.0470 - accuracy: 0.9070 - recall: 0.3013 - precision: 0.7128 - AUROC: 0.8195 - AUPRC: 0.5192 - f1_score: 0.4236 - balanced_accuracy: 0.6429 - specificity: 0.9845 - miss_rate: 0.6987 - fall_out: 0.0155 - mcc: 0.4242 - val_loss: 2.0532 - val_accuracy: 0.9127 - val_recall: 0.2815 - val_precision: 0.8452 - val_AUROC: 0.8518 - val_AUPRC: 0.5899 - val_f1_score: 0.4224 - val_balanced_accuracy: 0.6375 - val_specificity: 0.9934 - val_miss_rate: 0.7185 - val_fall_out: 0.0066 - val_mcc: 0.4573
Epoch 3/1000
198/198 [==============================] - 12s 60ms/step - loss: 1.8130 - accuracy: 0.9078 - recall: 0.3093 - precision: 0.7164 - AUROC: 0.8375 - AUPRC: 0.5423 - f1_score: 0.4321 - balanced_accuracy: 0.6468 - specificity: 0.9843 - miss_rate: 0.6907 - fall_out: 0.0157 - mcc: 0.4315 - val_loss: 1.9805 - val_accuracy: 0.8922 - val_recall: 0.0523 - val_precision: 0.9375 - val_AUROC: 0.8558 - val_AUPRC: 0.5740 - val_f1_score: 0.0990 - val_balanced_accuracy: 0.5259 - val_specificity: 0.9996 - val_miss_rate: 0.9477 - val_fall_out: 4.4555e-04 - val_mcc: 0.2073
Epoch 4/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.6940 - accuracy: 0.9105 - recall: 0.3260 - precision: 0.7385 - AUROC: 0.8439 - AUPRC: 0.5573 - f1_score: 0.4523 - balanced_accuracy: 0.6556 - specificity: 0.9852 - miss_rate: 0.6740 - fall_out: 0.0148 - mcc: 0.4525 - val_loss: 1.8574 - val_accuracy: 0.8969 - val_recall: 0.0962 - val_precision: 0.9452 - val_AUROC: 0.8612 - val_AUPRC: 0.5908 - val_f1_score: 0.1746 - val_balanced_accuracy: 0.5477 - val_specificity: 0.9993 - val_miss_rate: 0.9038 - val_fall_out: 7.1289e-04 - val_mcc: 0.2834
Epoch 5/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.6000 - accuracy: 0.9117 - recall: 0.3328 - precision: 0.7488 - AUROC: 0.8492 - AUPRC: 0.5707 - f1_score: 0.4608 - balanced_accuracy: 0.6593 - specificity: 0.9857 - miss_rate: 0.6672 - fall_out: 0.0143 - mcc: 0.4617 - val_loss: 1.7289 - val_accuracy: 0.9053 - val_recall: 0.1819 - val_precision: 0.9126 - val_AUROC: 0.8618 - val_AUPRC: 0.6006 - val_f1_score: 0.3033 - val_balanced_accuracy: 0.5898 - val_specificity: 0.9978 - val_miss_rate: 0.8181 - val_fall_out: 0.0022 - val_mcc: 0.3833
Epoch 6/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.5271 - accuracy: 0.9125 - recall: 0.3405 - precision: 0.7522 - AUROC: 0.8552 - AUPRC: 0.5778 - f1_score: 0.4688 - balanced_accuracy: 0.6631 - specificity: 0.9857 - miss_rate: 0.6595 - fall_out: 0.0143 - mcc: 0.4686 - val_loss: 1.7251 - val_accuracy: 0.9065 - val_recall: 0.1902 - val_precision: 0.9254 - val_AUROC: 0.8625 - val_AUPRC: 0.6053 - val_f1_score: 0.3156 - val_balanced_accuracy: 0.5941 - val_specificity: 0.9980 - val_miss_rate: 0.8098 - val_fall_out: 0.0020 - val_mcc: 0.3957
Epoch 7/1000
198/198 [==============================] - 12s 60ms/step - loss: 1.4665 - accuracy: 0.9133 - recall: 0.3436 - precision: 0.7606 - AUROC: 0.8587 - AUPRC: 0.5875 - f1_score: 0.4734 - balanced_accuracy: 0.6649 - specificity: 0.9862 - miss_rate: 0.6564 - fall_out: 0.0138 - mcc: 0.4743 - val_loss: 1.6684 - val_accuracy: 0.9078 - val_recall: 0.2084 - val_precision: 0.9061 - val_AUROC: 0.8601 - val_AUPRC: 0.6009 - val_f1_score: 0.3388 - val_balanced_accuracy: 0.6028 - val_specificity: 0.9972 - val_miss_rate: 0.7916 - val_fall_out: 0.0028 - val_mcc: 0.4091
Epoch 8/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.4150 - accuracy: 0.9136 - recall: 0.3499 - precision: 0.7584 - AUROC: 0.8661 - AUPRC: 0.5940 - f1_score: 0.4788 - balanced_accuracy: 0.6678 - specificity: 0.9857 - miss_rate: 0.6501 - fall_out: 0.0143 - mcc: 0.4779 - val_loss: 1.5873 - val_accuracy: 0.9078 - val_recall: 0.2077 - val_precision: 0.9085 - val_AUROC: 0.8595 - val_AUPRC: 0.6036 - val_f1_score: 0.3381 - val_balanced_accuracy: 0.6025 - val_specificity: 0.9973 - val_miss_rate: 0.7923 - val_fall_out: 0.0027 - val_mcc: 0.4091
Epoch 9/1000
198/198 [==============================] - 12s 60ms/step - loss: 1.3734 - accuracy: 0.9145 - recall: 0.3570 - precision: 0.7626 - AUROC: 0.8677 - AUPRC: 0.5973 - f1_score: 0.4864 - balanced_accuracy: 0.6714 - specificity: 0.9858 - miss_rate: 0.6430 - fall_out: 0.0142 - mcc: 0.4848 - val_loss: 1.5604 - val_accuracy: 0.9022 - val_recall: 0.1463 - val_precision: 0.9417 - val_AUROC: 0.8600 - val_AUPRC: 0.5954 - val_f1_score: 0.2533 - val_balanced_accuracy: 0.5726 - val_specificity: 0.9988 - val_miss_rate: 0.8537 - val_fall_out: 0.0012 - val_mcc: 0.3499
Epoch 10/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.3353 - accuracy: 0.9148 - recall: 0.3614 - precision: 0.7617 - AUROC: 0.8720 - AUPRC: 0.6029 - f1_score: 0.4902 - balanced_accuracy: 0.6735 - specificity: 0.9855 - miss_rate: 0.6386 - fall_out: 0.0145 - mcc: 0.4875 - val_loss: 1.5536 - val_accuracy: 0.9119 - val_recall: 0.2613 - val_precision: 0.8721 - val_AUROC: 0.8607 - val_AUPRC: 0.6033 - val_f1_score: 0.4021 - val_balanced_accuracy: 0.6282 - val_specificity: 0.9951 - val_miss_rate: 0.7387 - val_fall_out: 0.0049 - val_mcc: 0.4488
Epoch 11/1000
198/198 [==============================] - 12s 60ms/step - loss: 1.3022 - accuracy: 0.9150 - recall: 0.3631 - precision: 0.7637 - AUROC: 0.8739 - AUPRC: 0.6091 - f1_score: 0.4922 - balanced_accuracy: 0.6744 - specificity: 0.9856 - miss_rate: 0.6369 - fall_out: 0.0144 - mcc: 0.4896 - val_loss: 1.5000 - val_accuracy: 0.9136 - val_recall: 0.2948 - val_precision: 0.8376 - val_AUROC: 0.8601 - val_AUPRC: 0.6051 - val_f1_score: 0.4361 - val_balanced_accuracy: 0.6437 - val_specificity: 0.9927 - val_miss_rate: 0.7052 - val_fall_out: 0.0073 - val_mcc: 0.4657
Epoch 12/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.2716 - accuracy: 0.9148 - recall: 0.3610 - precision: 0.7627 - AUROC: 0.8767 - AUPRC: 0.6118 - f1_score: 0.4901 - balanced_accuracy: 0.6733 - specificity: 0.9856 - miss_rate: 0.6390 - fall_out: 0.0144 - mcc: 0.4877 - val_loss: 1.4713 - val_accuracy: 0.9134 - val_recall: 0.2836 - val_precision: 0.8568 - val_AUROC: 0.8551 - val_AUPRC: 0.6031 - val_f1_score: 0.4262 - val_balanced_accuracy: 0.6388 - val_specificity: 0.9939 - val_miss_rate: 0.7164 - val_fall_out: 0.0061 - val_mcc: 0.4630
Epoch 13/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.2436 - accuracy: 0.9149 - recall: 0.3699 - precision: 0.7551 - AUROC: 0.8801 - AUPRC: 0.6184 - f1_score: 0.4966 - balanced_accuracy: 0.6773 - specificity: 0.9847 - miss_rate: 0.6301 - fall_out: 0.0153 - mcc: 0.4908 - val_loss: 1.4291 - val_accuracy: 0.9132 - val_recall: 0.2836 - val_precision: 0.8532 - val_AUROC: 0.8571 - val_AUPRC: 0.6027 - val_f1_score: 0.4257 - val_balanced_accuracy: 0.6387 - val_specificity: 0.9938 - val_miss_rate: 0.7164 - val_fall_out: 0.0062 - val_mcc: 0.4618
Epoch 14/1000
198/198 [==============================] - 12s 60ms/step - loss: 1.2181 - accuracy: 0.9153 - recall: 0.3729 - precision: 0.7571 - AUROC: 0.8834 - AUPRC: 0.6251 - f1_score: 0.4996 - balanced_accuracy: 0.6788 - specificity: 0.9847 - miss_rate: 0.6271 - fall_out: 0.0153 - mcc: 0.4937 - val_loss: 1.3798 - val_accuracy: 0.9120 - val_recall: 0.2599 - val_precision: 0.8776 - val_AUROC: 0.8572 - val_AUPRC: 0.6016 - val_f1_score: 0.4011 - val_balanced_accuracy: 0.6276 - val_specificity: 0.9954 - val_miss_rate: 0.7401 - val_fall_out: 0.0046 - val_mcc: 0.4493
Epoch 15/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.1936 - accuracy: 0.9156 - recall: 0.3767 - precision: 0.7566 - AUROC: 0.8860 - AUPRC: 0.6275 - f1_score: 0.5030 - balanced_accuracy: 0.6806 - specificity: 0.9845 - miss_rate: 0.6233 - fall_out: 0.0155 - mcc: 0.4962 - val_loss: 1.3720 - val_accuracy: 0.9106 - val_recall: 0.2362 - val_precision: 0.9040 - val_AUROC: 0.8567 - val_AUPRC: 0.6063 - val_f1_score: 0.3746 - val_balanced_accuracy: 0.6165 - val_specificity: 0.9968 - val_miss_rate: 0.7638 - val_fall_out: 0.0032 - val_mcc: 0.4357
Epoch 16/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.1704 - accuracy: 0.9165 - recall: 0.3861 - precision: 0.7600 - AUROC: 0.8887 - AUPRC: 0.6348 - f1_score: 0.5121 - balanced_accuracy: 0.6853 - specificity: 0.9844 - miss_rate: 0.6139 - fall_out: 0.0156 - mcc: 0.5042 - val_loss: 1.2936 - val_accuracy: 0.9136 - val_recall: 0.2857 - val_precision: 0.8559 - val_AUROC: 0.8504 - val_AUPRC: 0.6026 - val_f1_score: 0.4284 - val_balanced_accuracy: 0.6398 - val_specificity: 0.9939 - val_miss_rate: 0.7143 - val_fall_out: 0.0061 - val_mcc: 0.4645
Epoch 17/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.1480 - accuracy: 0.9168 - recall: 0.3932 - precision: 0.7564 - AUROC: 0.8907 - AUPRC: 0.6422 - f1_score: 0.5175 - balanced_accuracy: 0.6885 - specificity: 0.9838 - miss_rate: 0.6068 - fall_out: 0.0162 - mcc: 0.5076 - val_loss: 1.2636 - val_accuracy: 0.9142 - val_recall: 0.2941 - val_precision: 0.8525 - val_AUROC: 0.8506 - val_AUPRC: 0.6025 - val_f1_score: 0.4373 - val_balanced_accuracy: 0.6438 - val_specificity: 0.9935 - val_miss_rate: 0.7059 - val_fall_out: 0.0065 - val_mcc: 0.4703
Epoch 18/1000
198/198 [==============================] - 12s 60ms/step - loss: 1.1261 - accuracy: 0.9166 - recall: 0.3997 - precision: 0.7473 - AUROC: 0.8954 - AUPRC: 0.6468 - f1_score: 0.5208 - balanced_accuracy: 0.6912 - specificity: 0.9827 - miss_rate: 0.6003 - fall_out: 0.0173 - mcc: 0.5080 - val_loss: 1.2228 - val_accuracy: 0.9139 - val_recall: 0.2836 - val_precision: 0.8678 - val_AUROC: 0.8548 - val_AUPRC: 0.6018 - val_f1_score: 0.4275 - val_balanced_accuracy: 0.6390 - val_specificity: 0.9945 - val_miss_rate: 0.7164 - val_fall_out: 0.0055 - val_mcc: 0.4668
Epoch 19/1000
198/198 [==============================] - 12s 60ms/step - loss: 1.1039 - accuracy: 0.9166 - recall: 0.4107 - precision: 0.7383 - AUROC: 0.8992 - AUPRC: 0.6549 - f1_score: 0.5278 - balanced_accuracy: 0.6960 - specificity: 0.9814 - miss_rate: 0.5893 - fall_out: 0.0186 - mcc: 0.5113 - val_loss: 1.2133 - val_accuracy: 0.9164 - val_recall: 0.3401 - val_precision: 0.8147 - val_AUROC: 0.8415 - val_AUPRC: 0.5971 - val_f1_score: 0.4798 - val_balanced_accuracy: 0.6651 - val_specificity: 0.9901 - val_miss_rate: 0.6599 - val_fall_out: 0.0099 - val_mcc: 0.4930
Epoch 20/1000
198/198 [==============================] - 12s 60ms/step - loss: 1.0847 - accuracy: 0.9169 - recall: 0.4230 - precision: 0.7305 - AUROC: 0.9011 - AUPRC: 0.6608 - f1_score: 0.5358 - balanced_accuracy: 0.7015 - specificity: 0.9800 - miss_rate: 0.5770 - fall_out: 0.0200 - mcc: 0.5160 - val_loss: 1.1467 - val_accuracy: 0.9157 - val_recall: 0.3303 - val_precision: 0.8172 - val_AUROC: 0.8511 - val_AUPRC: 0.5989 - val_f1_score: 0.4705 - val_balanced_accuracy: 0.6604 - val_specificity: 0.9906 - val_miss_rate: 0.6697 - val_fall_out: 0.0094 - val_mcc: 0.4865
Epoch 21/1000
198/198 [==============================] - 12s 59ms/step - loss: 1.0655 - accuracy: 0.9181 - recall: 0.4293 - precision: 0.7391 - AUROC: 0.9036 - AUPRC: 0.6651 - f1_score: 0.5431 - balanced_accuracy: 0.7050 - specificity: 0.9806 - miss_rate: 0.5707 - fall_out: 0.0194 - mcc: 0.5240 - val_loss: 1.1124 - val_accuracy: 0.9178 - val_recall: 0.3847 - val_precision: 0.7775 - val_AUROC: 0.8445 - val_AUPRC: 0.5974 - val_f1_score: 0.5147 - val_balanced_accuracy: 0.6853 - val_specificity: 0.9859 - val_miss_rate: 0.6153 - val_fall_out: 0.0141 - val_mcc: 0.5106
Epoch 22/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.0478 - accuracy: 0.9178 - recall: 0.4357 - precision: 0.7305 - AUROC: 0.9050 - AUPRC: 0.6672 - f1_score: 0.5459 - balanced_accuracy: 0.7076 - specificity: 0.9794 - miss_rate: 0.5643 - fall_out: 0.0206 - mcc: 0.5242 - val_loss: 1.1215 - val_accuracy: 0.9177 - val_recall: 0.3693 - val_precision: 0.7946 - val_AUROC: 0.8475 - val_AUPRC: 0.5977 - val_f1_score: 0.5043 - val_balanced_accuracy: 0.6786 - val_specificity: 0.9878 - val_miss_rate: 0.6307 - val_fall_out: 0.0122 - val_mcc: 0.5068
Epoch 23/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.0266 - accuracy: 0.9192 - recall: 0.4617 - precision: 0.7261 - AUROC: 0.9101 - AUPRC: 0.6788 - f1_score: 0.5645 - balanced_accuracy: 0.7197 - specificity: 0.9777 - miss_rate: 0.5383 - fall_out: 0.0223 - mcc: 0.5386 - val_loss: 1.1100 - val_accuracy: 0.9172 - val_recall: 0.3840 - val_precision: 0.7706 - val_AUROC: 0.8371 - val_AUPRC: 0.5932 - val_f1_score: 0.5126 - val_balanced_accuracy: 0.6847 - val_specificity: 0.9854 - val_miss_rate: 0.6160 - val_fall_out: 0.0146 - val_mcc: 0.5072
Epoch 24/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.0094 - accuracy: 0.9206 - recall: 0.4761 - precision: 0.7298 - AUROC: 0.9118 - AUPRC: 0.6826 - f1_score: 0.5763 - balanced_accuracy: 0.7268 - specificity: 0.9775 - miss_rate: 0.5239 - fall_out: 0.0225 - mcc: 0.5495 - val_loss: 1.1130 - val_accuracy: 0.9159 - val_recall: 0.4042 - val_precision: 0.7351 - val_AUROC: 0.8445 - val_AUPRC: 0.5959 - val_f1_score: 0.5216 - val_balanced_accuracy: 0.6928 - val_specificity: 0.9814 - val_miss_rate: 0.5958 - val_fall_out: 0.0186 - val_mcc: 0.5056
Epoch 25/1000
198/198 [==============================] - 13s 60ms/step - loss: 0.9910 - accuracy: 0.9224 - recall: 0.5136 - precision: 0.7223 - AUROC: 0.9165 - AUPRC: 0.6938 - f1_score: 0.6003 - balanced_accuracy: 0.7442 - specificity: 0.9747 - miss_rate: 0.4864 - fall_out: 0.0253 - mcc: 0.5687 - val_loss: 1.0481 - val_accuracy: 0.9163 - val_recall: 0.3582 - val_precision: 0.7883 - val_AUROC: 0.8443 - val_AUPRC: 0.5934 - val_f1_score: 0.4926 - val_balanced_accuracy: 0.6729 - val_specificity: 0.9877 - val_miss_rate: 0.6418 - val_fall_out: 0.0123 - val_mcc: 0.4961
Epoch 26/1000
198/198 [==============================] - 12s 60ms/step - loss: 0.9753 - accuracy: 0.9227 - recall: 0.5221 - precision: 0.7195 - AUROC: 0.9175 - AUPRC: 0.6951 - f1_score: 0.6051 - balanced_accuracy: 0.7480 - specificity: 0.9740 - miss_rate: 0.4779 - fall_out: 0.0260 - mcc: 0.5724 - val_loss: 1.0203 - val_accuracy: 0.9163 - val_recall: 0.3805 - val_precision: 0.7615 - val_AUROC: 0.8365 - val_AUPRC: 0.5865 - val_f1_score: 0.5074 - val_balanced_accuracy: 0.6826 - val_specificity: 0.9848 - val_miss_rate: 0.6195 - val_fall_out: 0.0152 - val_mcc: 0.5009
Epoch 27/1000
198/198 [==============================] - 13s 60ms/step - loss: 0.9582 - accuracy: 0.9244 - recall: 0.5531 - precision: 0.7155 - AUROC: 0.9209 - AUPRC: 0.7040 - f1_score: 0.6239 - balanced_accuracy: 0.7625 - specificity: 0.9719 - miss_rate: 0.4469 - fall_out: 0.0281 - mcc: 0.5886 - val_loss: 1.0486 - val_accuracy: 0.9151 - val_recall: 0.3951 - val_precision: 0.7335 - val_AUROC: 0.8397 - val_AUPRC: 0.5861 - val_f1_score: 0.5136 - val_balanced_accuracy: 0.6884 - val_specificity: 0.9816 - val_miss_rate: 0.6049 - val_fall_out: 0.0184 - val_mcc: 0.4988
Epoch 28/1000
198/198 [==============================] - 13s 60ms/step - loss: 0.9410 - accuracy: 0.9251 - recall: 0.5711 - precision: 0.7119 - AUROC: 0.9238 - AUPRC: 0.7102 - f1_score: 0.6337 - balanced_accuracy: 0.7707 - specificity: 0.9704 - miss_rate: 0.4289 - fall_out: 0.0296 - mcc: 0.5971 - val_loss: 1.0273 - val_accuracy: 0.9156 - val_recall: 0.3756 - val_precision: 0.7581 - val_AUROC: 0.8396 - val_AUPRC: 0.5901 - val_f1_score: 0.5023 - val_balanced_accuracy: 0.6801 - val_specificity: 0.9847 - val_miss_rate: 0.6244 - val_fall_out: 0.0153 - val_mcc: 0.4961
198/198 [==============================] - 7s 35ms/step - loss: 0.9369 - accuracy: 0.9281 - recall: 0.4528 - precision: 0.8393 - AUROC: 0.9539 - AUPRC: 0.7950 - f1_score: 0.5882 - balanced_accuracy: 0.7209 - specificity: 0.9889 - miss_rate: 0.5472 - fall_out: 0.0111 - mcc: 0.5844
50/50 [==============================] - 4s 83ms/step - loss: 1.0273 - accuracy: 0.9156 - recall: 0.3756 - precision: 0.7581 - AUROC: 0.8396 - AUPRC: 0.5901 - f1_score: 0.5023 - balanced_accuracy: 0.6801 - specificity: 0.9847 - miss_rate: 0.6244 - fall_out: 0.0153 - mcc: 0.4961
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_0\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_0\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 4, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 72, 'dropout_1': 0.5, 'units_2': 136, 'dropout_2': 0.5, 'units_3': 8, 'dropout_3': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_0\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_0\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 4, 'filters_0': 160, 'kernel_size_0': 8, 'conv_dropout_0': 0.2, 'dense_depth': 5, 'units_0': 128, 'dense_dropout_0': 0.5, 'filters_1': 160, 'kernel_size_1': 16, 'conv_dropout_1': 0.1, 'filters_2': 24, 'kernel_size_2': 12, 'conv_dropout_2': 0.1, 'filters_3': 152, 'kernel_size_3': 16, 'conv_dropout_3': 0.0, 'filters_4': 160, 'kernel_size_4': 16, 'conv_dropout_4': 0.2, 'units_1': 96, 'dense_dropout_1': 0.4, 'units_2': 64, 'dense_dropout_2': 0.3, 'units_3': 128, 'dense_dropout_3': 0.5, 'units_4': 112, 'dense_dropout_4': 0.4, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['SNRNP70', 'ZNF282']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 72) 9864
dropout_4 (Dropout) (None, 72) 0
dense_6 (Dense) (None, 136) 9928
dropout_5 (Dropout) (None, 136) 0
dense_7 (Dense) (None, 8) 1096
dropout_6 (Dropout) (None, 8) 0
dense_8 (Dense) (None, 1) 9
=================================================================
Total params: 97,329
Trainable params: 97,329
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 5280
dropout_7 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 160) 409760
dropout_8 (Dropout) (None, 128, 160) 0
max_pooling1d_2 (MaxPooling (None, 64, 160) 0
1D)
conv1d_3 (Conv1D) (None, 64, 24) 46104
dropout_9 (Dropout) (None, 64, 24) 0
max_pooling1d_3 (MaxPooling (None, 32, 24) 0
1D)
conv1d_4 (Conv1D) (None, 32, 152) 58520
dropout_10 (Dropout) (None, 32, 152) 0
max_pooling1d_4 (MaxPooling (None, 16, 152) 0
1D)
flatten_1 (Flatten) (None, 2432) 0
dense_9 (Dense) (None, 128) 311424
dropout_11 (Dropout) (None, 128) 0
dense_10 (Dense) (None, 96) 12384
dropout_12 (Dropout) (None, 96) 0
dense_11 (Dense) (None, 64) 6208
dropout_13 (Dropout) (None, 64) 0
dense_12 (Dense) (None, 128) 8320
dropout_14 (Dropout) (None, 128) 0
dense_13 (Dense) (None, 112) 14448
dropout_15 (Dropout) (None, 112) 0
dense_14 (Dense) (None, 1) 113
=================================================================
Total params: 872,561
Trainable params: 872,561
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 5280 ['sequence_data[0][0]']
dropout_7 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_7[0][0]']
conv1d_2 (Conv1D) (None, 128, 160) 409760 ['max_pooling1d_1[0][0]']
dropout_8 (Dropout) (None, 128, 160) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 160) 0 ['dropout_8[0][0]']
conv1d_3 (Conv1D) (None, 64, 24) 46104 ['max_pooling1d_2[0][0]']
dropout_9 (Dropout) (None, 64, 24) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 24) 0 ['dropout_9[0][0]']
conv1d_4 (Conv1D) (None, 32, 152) 58520 ['max_pooling1d_3[0][0]']
dropout_10 (Dropout) (None, 32, 152) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 152) 0 ['dropout_10[0][0]']
flatten_1 (Flatten) (None, 2432) 0 ['max_pooling1d_4[0][0]']
dense_9 (Dense) (None, 128) 311424 ['flatten_1[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
dropout_11 (Dropout) (None, 128) 0 ['dense_9[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
dense_10 (Dense) (None, 96) 12384 ['dropout_11[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_12 (Dropout) (None, 96) 0 ['dense_10[0][0]']
dense_5 (Dense) (None, 72) 9864 ['dropout_3[0][0]']
dense_11 (Dense) (None, 64) 6208 ['dropout_12[0][0]']
dropout_4 (Dropout) (None, 72) 0 ['dense_5[0][0]']
dropout_13 (Dropout) (None, 64) 0 ['dense_11[0][0]']
dense_6 (Dense) (None, 136) 9928 ['dropout_4[0][0]']
dense_12 (Dense) (None, 128) 8320 ['dropout_13[0][0]']
dropout_5 (Dropout) (None, 136) 0 ['dense_6[0][0]']
dropout_14 (Dropout) (None, 128) 0 ['dense_12[0][0]']
dense_7 (Dense) (None, 8) 1096 ['dropout_5[0][0]']
dense_13 (Dense) (None, 112) 14448 ['dropout_14[0][0]']
dropout_6 (Dropout) (None, 8) 0 ['dense_7[0][0]']
dropout_15 (Dropout) (None, 112) 0 ['dense_13[0][0]']
concatenate (Concatenate) (None, 120) 0 ['dropout_6[0][0]',
'dropout_15[0][0]']
dense_15 (Dense) (None, 64) 7744 ['concatenate[0][0]']
dense_16 (Dense) (None, 1) 65 ['dense_15[0][0]']
==================================================================================================
Total params: 977,577
Trainable params: 977,577
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 15s 66ms/step - loss: 0.3268 - accuracy: 0.8921 - recall: 0.2203 - precision: 0.5620 - AUROC: 0.7089 - AUPRC: 0.3654 - f1_score: 0.3165 - balanced_accuracy: 0.5992 - specificity: 0.9780 - miss_rate: 0.7797 - fall_out: 0.0220 - mcc: 0.3051 - val_loss: 0.2568 - val_accuracy: 0.9061 - val_recall: 0.1965 - val_precision: 0.8896 - val_AUROC: 0.8402 - val_AUPRC: 0.5782 - val_f1_score: 0.3219 - val_balanced_accuracy: 0.5967 - val_specificity: 0.9969 - val_miss_rate: 0.8035 - val_fall_out: 0.0031 - val_mcc: 0.3924
Epoch 2/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.2674 - accuracy: 0.9071 - recall: 0.2867 - precision: 0.7306 - AUROC: 0.8133 - AUPRC: 0.5142 - f1_score: 0.4118 - balanced_accuracy: 0.6366 - specificity: 0.9865 - miss_rate: 0.7133 - fall_out: 0.0135 - mcc: 0.4200 - val_loss: 0.2556 - val_accuracy: 0.9000 - val_recall: 0.1233 - val_precision: 0.9568 - val_AUROC: 0.8604 - val_AUPRC: 0.5976 - val_f1_score: 0.2185 - val_balanced_accuracy: 0.5613 - val_specificity: 0.9993 - val_miss_rate: 0.8767 - val_fall_out: 7.1289e-04 - val_mcc: 0.3240
Epoch 3/1000
198/198 [==============================] - 13s 63ms/step - loss: 0.2586 - accuracy: 0.9078 - recall: 0.3015 - precision: 0.7255 - AUROC: 0.8312 - AUPRC: 0.5413 - f1_score: 0.4259 - balanced_accuracy: 0.6434 - specificity: 0.9854 - miss_rate: 0.6985 - fall_out: 0.0146 - mcc: 0.4293 - val_loss: 0.2467 - val_accuracy: 0.9108 - val_recall: 0.2453 - val_precision: 0.8844 - val_AUROC: 0.8572 - val_AUPRC: 0.6013 - val_f1_score: 0.3841 - val_balanced_accuracy: 0.6206 - val_specificity: 0.9959 - val_miss_rate: 0.7547 - val_fall_out: 0.0041 - val_mcc: 0.4382
Epoch 4/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.2534 - accuracy: 0.9100 - recall: 0.3175 - precision: 0.7411 - AUROC: 0.8374 - AUPRC: 0.5555 - f1_score: 0.4445 - balanced_accuracy: 0.6516 - specificity: 0.9858 - miss_rate: 0.6825 - fall_out: 0.0142 - mcc: 0.4473 - val_loss: 0.2439 - val_accuracy: 0.9123 - val_recall: 0.2620 - val_precision: 0.8806 - val_AUROC: 0.8605 - val_AUPRC: 0.6050 - val_f1_score: 0.4039 - val_balanced_accuracy: 0.6287 - val_specificity: 0.9955 - val_miss_rate: 0.7380 - val_fall_out: 0.0045 - val_mcc: 0.4521
Epoch 5/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.2479 - accuracy: 0.9113 - recall: 0.3276 - precision: 0.7485 - AUROC: 0.8484 - AUPRC: 0.5704 - f1_score: 0.4557 - balanced_accuracy: 0.6568 - specificity: 0.9859 - miss_rate: 0.6724 - fall_out: 0.0141 - mcc: 0.4577 - val_loss: 0.2409 - val_accuracy: 0.9118 - val_recall: 0.2564 - val_precision: 0.8825 - val_AUROC: 0.8644 - val_AUPRC: 0.6072 - val_f1_score: 0.3974 - val_balanced_accuracy: 0.6260 - val_specificity: 0.9956 - val_miss_rate: 0.7436 - val_fall_out: 0.0044 - val_mcc: 0.4478
Epoch 6/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.2458 - accuracy: 0.9115 - recall: 0.3258 - precision: 0.7535 - AUROC: 0.8538 - AUPRC: 0.5734 - f1_score: 0.4550 - balanced_accuracy: 0.6561 - specificity: 0.9864 - miss_rate: 0.6742 - fall_out: 0.0136 - mcc: 0.4584 - val_loss: 0.2452 - val_accuracy: 0.9097 - val_recall: 0.2272 - val_precision: 0.9056 - val_AUROC: 0.8626 - val_AUPRC: 0.6065 - val_f1_score: 0.3632 - val_balanced_accuracy: 0.6121 - val_specificity: 0.9970 - val_miss_rate: 0.7728 - val_fall_out: 0.0030 - val_mcc: 0.4275
Epoch 7/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.2433 - accuracy: 0.9115 - recall: 0.3299 - precision: 0.7501 - AUROC: 0.8565 - AUPRC: 0.5815 - f1_score: 0.4582 - balanced_accuracy: 0.6579 - specificity: 0.9859 - miss_rate: 0.6701 - fall_out: 0.0141 - mcc: 0.4600 - val_loss: 0.2458 - val_accuracy: 0.9075 - val_recall: 0.2021 - val_precision: 0.9177 - val_AUROC: 0.8609 - val_AUPRC: 0.6083 - val_f1_score: 0.3312 - val_balanced_accuracy: 0.5999 - val_specificity: 0.9977 - val_miss_rate: 0.7979 - val_fall_out: 0.0023 - val_mcc: 0.4060
198/198 [==============================] - 7s 33ms/step - loss: 0.2369 - accuracy: 0.9083 - recall: 0.2090 - precision: 0.9217 - AUROC: 0.8809 - AUPRC: 0.6303 - f1_score: 0.3407 - balanced_accuracy: 0.6034 - specificity: 0.9977 - miss_rate: 0.7910 - fall_out: 0.0023 - mcc: 0.4141
50/50 [==============================] - 5s 88ms/step - loss: 0.2458 - accuracy: 0.9075 - recall: 0.2021 - precision: 0.9177 - AUROC: 0.8609 - AUPRC: 0.6083 - f1_score: 0.3312 - balanced_accuracy: 0.5999 - specificity: 0.9977 - miss_rate: 0.7979 - fall_out: 0.0023 - mcc: 0.4060
-- HOLDOUT 2 --
-- 1 Uncorrelated features: [Pearson+Spearman]
['SNRNP70']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 23s 62ms/step - loss: 18.0544 - accuracy: 0.8965 - recall: 0.2344 - precision: 0.6152 - AUROC: 0.7367 - AUPRC: 0.3827 - f1_score: 0.3395 - balanced_accuracy: 0.6078 - specificity: 0.9812 - miss_rate: 0.7656 - fall_out: 0.0188 - mcc: 0.3363 - val_loss: 2.3297 - val_accuracy: 0.9105 - val_recall: 0.2704 - val_precision: 0.8186 - val_AUROC: 0.8418 - val_AUPRC: 0.5750 - val_f1_score: 0.4065 - val_balanced_accuracy: 0.6314 - val_specificity: 0.9923 - val_miss_rate: 0.7296 - val_fall_out: 0.0077 - val_mcc: 0.4387
Epoch 2/1000
198/198 [==============================] - 13s 60ms/step - loss: 2.0787 - accuracy: 0.9069 - recall: 0.3030 - precision: 0.7093 - AUROC: 0.8200 - AUPRC: 0.5139 - f1_score: 0.4246 - balanced_accuracy: 0.6436 - specificity: 0.9841 - miss_rate: 0.6970 - fall_out: 0.0159 - mcc: 0.4241 - val_loss: 1.6968 - val_accuracy: 0.9037 - val_recall: 0.1596 - val_precision: 0.9463 - val_AUROC: 0.8556 - val_AUPRC: 0.5928 - val_f1_score: 0.2731 - val_balanced_accuracy: 0.5792 - val_specificity: 0.9988 - val_miss_rate: 0.8404 - val_fall_out: 0.0012 - val_mcc: 0.3668
Epoch 3/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.8304 - accuracy: 0.9093 - recall: 0.3220 - precision: 0.7260 - AUROC: 0.8343 - AUPRC: 0.5416 - f1_score: 0.4461 - balanced_accuracy: 0.6532 - specificity: 0.9844 - miss_rate: 0.6780 - fall_out: 0.0156 - mcc: 0.4446 - val_loss: 1.5233 - val_accuracy: 0.9072 - val_recall: 0.2049 - val_precision: 0.8963 - val_AUROC: 0.8607 - val_AUPRC: 0.5913 - val_f1_score: 0.3335 - val_balanced_accuracy: 0.6009 - val_specificity: 0.9970 - val_miss_rate: 0.7951 - val_fall_out: 0.0030 - val_mcc: 0.4028
Epoch 4/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.7068 - accuracy: 0.9111 - recall: 0.3302 - precision: 0.7435 - AUROC: 0.8444 - AUPRC: 0.5617 - f1_score: 0.4573 - balanced_accuracy: 0.6578 - specificity: 0.9854 - miss_rate: 0.6698 - fall_out: 0.0146 - mcc: 0.4576 - val_loss: 1.4199 - val_accuracy: 0.9077 - val_recall: 0.2070 - val_precision: 0.9083 - val_AUROC: 0.8606 - val_AUPRC: 0.6023 - val_f1_score: 0.3371 - val_balanced_accuracy: 0.6021 - val_specificity: 0.9973 - val_miss_rate: 0.7930 - val_fall_out: 0.0027 - val_mcc: 0.4083
Epoch 5/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.6115 - accuracy: 0.9114 - recall: 0.3370 - precision: 0.7402 - AUROC: 0.8527 - AUPRC: 0.5686 - f1_score: 0.4631 - balanced_accuracy: 0.6609 - specificity: 0.9849 - miss_rate: 0.6630 - fall_out: 0.0151 - mcc: 0.4612 - val_loss: 1.3454 - val_accuracy: 0.9127 - val_recall: 0.2641 - val_precision: 0.8855 - val_AUROC: 0.8612 - val_AUPRC: 0.6063 - val_f1_score: 0.4069 - val_balanced_accuracy: 0.6299 - val_specificity: 0.9956 - val_miss_rate: 0.7359 - val_fall_out: 0.0044 - val_mcc: 0.4556
Epoch 6/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.5363 - accuracy: 0.9123 - recall: 0.3391 - precision: 0.7514 - AUROC: 0.8570 - AUPRC: 0.5776 - f1_score: 0.4673 - balanced_accuracy: 0.6624 - specificity: 0.9857 - miss_rate: 0.6609 - fall_out: 0.0143 - mcc: 0.4673 - val_loss: 1.2643 - val_accuracy: 0.9044 - val_recall: 0.1652 - val_precision: 0.9518 - val_AUROC: 0.8623 - val_AUPRC: 0.6026 - val_f1_score: 0.2815 - val_balanced_accuracy: 0.5820 - val_specificity: 0.9989 - val_miss_rate: 0.8348 - val_fall_out: 0.0011 - val_mcc: 0.3746
Epoch 7/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.4757 - accuracy: 0.9132 - recall: 0.3457 - precision: 0.7565 - AUROC: 0.8586 - AUPRC: 0.5863 - f1_score: 0.4745 - balanced_accuracy: 0.6657 - specificity: 0.9858 - miss_rate: 0.6543 - fall_out: 0.0142 - mcc: 0.4741 - val_loss: 1.1907 - val_accuracy: 0.9076 - val_recall: 0.1979 - val_precision: 0.9373 - val_AUROC: 0.8625 - val_AUPRC: 0.6081 - val_f1_score: 0.3268 - val_balanced_accuracy: 0.5981 - val_specificity: 0.9983 - val_miss_rate: 0.8021 - val_fall_out: 0.0017 - val_mcc: 0.4070
Epoch 8/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.4243 - accuracy: 0.9135 - recall: 0.3483 - precision: 0.7590 - AUROC: 0.8625 - AUPRC: 0.5896 - f1_score: 0.4775 - balanced_accuracy: 0.6671 - specificity: 0.9859 - miss_rate: 0.6517 - fall_out: 0.0141 - mcc: 0.4771 - val_loss: 1.1707 - val_accuracy: 0.9109 - val_recall: 0.2383 - val_precision: 0.9072 - val_AUROC: 0.8595 - val_AUPRC: 0.6052 - val_f1_score: 0.3775 - val_balanced_accuracy: 0.6176 - val_specificity: 0.9969 - val_miss_rate: 0.7617 - val_fall_out: 0.0031 - val_mcc: 0.4387
Epoch 9/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.3789 - accuracy: 0.9144 - recall: 0.3567 - precision: 0.7616 - AUROC: 0.8689 - AUPRC: 0.5967 - f1_score: 0.4858 - balanced_accuracy: 0.6712 - specificity: 0.9857 - miss_rate: 0.6433 - fall_out: 0.0143 - mcc: 0.4841 - val_loss: 1.0862 - val_accuracy: 0.9095 - val_recall: 0.2188 - val_precision: 0.9290 - val_AUROC: 0.8630 - val_AUPRC: 0.6115 - val_f1_score: 0.3542 - val_balanced_accuracy: 0.6083 - val_specificity: 0.9979 - val_miss_rate: 0.7812 - val_fall_out: 0.0021 - val_mcc: 0.4261
Epoch 10/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.3416 - accuracy: 0.9143 - recall: 0.3600 - precision: 0.7569 - AUROC: 0.8698 - AUPRC: 0.6004 - f1_score: 0.4879 - balanced_accuracy: 0.6726 - specificity: 0.9852 - miss_rate: 0.6400 - fall_out: 0.0148 - mcc: 0.4845 - val_loss: 1.0704 - val_accuracy: 0.9144 - val_recall: 0.2850 - val_precision: 0.8777 - val_AUROC: 0.8631 - val_AUPRC: 0.6124 - val_f1_score: 0.4303 - val_balanced_accuracy: 0.6400 - val_specificity: 0.9949 - val_miss_rate: 0.7150 - val_fall_out: 0.0051 - val_mcc: 0.4713
Epoch 11/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.3048 - accuracy: 0.9146 - recall: 0.3676 - precision: 0.7529 - AUROC: 0.8750 - AUPRC: 0.6101 - f1_score: 0.4940 - balanced_accuracy: 0.6761 - specificity: 0.9846 - miss_rate: 0.6324 - fall_out: 0.0154 - mcc: 0.4883 - val_loss: 1.0632 - val_accuracy: 0.9129 - val_recall: 0.2613 - val_precision: 0.8993 - val_AUROC: 0.8625 - val_AUPRC: 0.6116 - val_f1_score: 0.4050 - val_balanced_accuracy: 0.6288 - val_specificity: 0.9963 - val_miss_rate: 0.7387 - val_fall_out: 0.0037 - val_mcc: 0.4575
Epoch 12/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.2722 - accuracy: 0.9157 - recall: 0.3701 - precision: 0.7655 - AUROC: 0.8797 - AUPRC: 0.6198 - f1_score: 0.4989 - balanced_accuracy: 0.6778 - specificity: 0.9855 - miss_rate: 0.6299 - fall_out: 0.0145 - mcc: 0.4953 - val_loss: 1.0100 - val_accuracy: 0.9140 - val_recall: 0.2760 - val_precision: 0.8899 - val_AUROC: 0.8597 - val_AUPRC: 0.6130 - val_f1_score: 0.4213 - val_balanced_accuracy: 0.6358 - val_specificity: 0.9956 - val_miss_rate: 0.7240 - val_fall_out: 0.0044 - val_mcc: 0.4675
Epoch 13/1000
198/198 [==============================] - 13s 60ms/step - loss: 1.2447 - accuracy: 0.9153 - recall: 0.3757 - precision: 0.7539 - AUROC: 0.8798 - AUPRC: 0.6198 - f1_score: 0.5015 - balanced_accuracy: 0.6800 - specificity: 0.9843 - miss_rate: 0.6243 - fall_out: 0.0157 - mcc: 0.4943 - val_loss: 1.0191 - val_accuracy: 0.9137 - val_recall: 0.2836 - val_precision: 0.8641 - val_AUROC: 0.8597 - val_AUPRC: 0.6076 - val_f1_score: 0.4271 - val_balanced_accuracy: 0.6390 - val_specificity: 0.9943 - val_miss_rate: 0.7164 - val_fall_out: 0.0057 - val_mcc: 0.4655
Epoch 14/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.2180 - accuracy: 0.9161 - recall: 0.3858 - precision: 0.7549 - AUROC: 0.8831 - AUPRC: 0.6256 - f1_score: 0.5106 - balanced_accuracy: 0.6849 - specificity: 0.9840 - miss_rate: 0.6142 - fall_out: 0.0160 - mcc: 0.5018 - val_loss: 1.0278 - val_accuracy: 0.9144 - val_recall: 0.2878 - val_precision: 0.8713 - val_AUROC: 0.8592 - val_AUPRC: 0.6105 - val_f1_score: 0.4327 - val_balanced_accuracy: 0.6412 - val_specificity: 0.9946 - val_miss_rate: 0.7122 - val_fall_out: 0.0054 - val_mcc: 0.4715
198/198 [==============================] - 7s 36ms/step - loss: 0.9983 - accuracy: 0.9150 - recall: 0.2954 - precision: 0.8675 - AUROC: 0.9068 - AUPRC: 0.6733 - f1_score: 0.4407 - balanced_accuracy: 0.6448 - specificity: 0.9942 - miss_rate: 0.7046 - fall_out: 0.0058 - mcc: 0.4766
50/50 [==============================] - 5s 93ms/step - loss: 1.0278 - accuracy: 0.9144 - recall: 0.2878 - precision: 0.8713 - AUROC: 0.8592 - AUPRC: 0.6105 - f1_score: 0.4327 - balanced_accuracy: 0.6412 - specificity: 0.9946 - miss_rate: 0.7122 - fall_out: 0.0054 - mcc: 0.4715
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_1\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_1\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 5, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 136, 'dropout_1': 0.3, 'units_2': 136, 'dropout_2': 0.5, 'units_3': 136, 'dropout_3': 0.5, 'units_4': 8, 'dropout_4': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_1\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_1\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 8, 'conv_dropout_0': 0.0, 'dense_depth': 1, 'units_0': 128, 'dense_dropout_0': 0.5, 'units_1': 128, 'dense_dropout_1': 0.3, 'units_2': 128, 'dense_dropout_2': 0.5, 'units_3': 128, 'dense_dropout_3': 0.5, 'filters_1': 128, 'kernel_size_1': 6, 'conv_dropout_1': 0.0, 'filters_2': 8, 'kernel_size_2': 16, 'conv_dropout_2': 0.2, 'filters_3': 8, 'kernel_size_3': 12, 'conv_dropout_3': 0.0, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.1, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.1}
- Removing uncorrelated/correlated features: ['SNRNP70']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 136) 18632
dropout_4 (Dropout) (None, 136) 0
dense_6 (Dense) (None, 136) 18632
dropout_5 (Dropout) (None, 136) 0
dense_7 (Dense) (None, 136) 18632
dropout_6 (Dropout) (None, 136) 0
dense_8 (Dense) (None, 8) 1096
dropout_7 (Dropout) (None, 8) 0
dense_9 (Dense) (None, 1) 9
=================================================================
Total params: 133,433
Trainable params: 133,433
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 5280
dropout_8 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 128) 123008
dropout_9 (Dropout) (None, 128, 128) 0
max_pooling1d_2 (MaxPooling (None, 64, 128) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 16392
dropout_10 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 776
dropout_11 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 8) 392
dropout_12 (Dropout) (None, 16, 8) 0
max_pooling1d_5 (MaxPooling (None, 8, 8) 0
1D)
conv1d_6 (Conv1D) (None, 8, 8) 392
dropout_13 (Dropout) (None, 8, 8) 0
max_pooling1d_6 (MaxPooling (None, 4, 8) 0
1D)
flatten_1 (Flatten) (None, 32) 0
dense_10 (Dense) (None, 128) 4224
dropout_14 (Dropout) (None, 128) 0
dense_11 (Dense) (None, 1) 129
=================================================================
Total params: 150,593
Trainable params: 150,593
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 5280 ['sequence_data[0][0]']
dropout_8 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_8[0][0]']
conv1d_2 (Conv1D) (None, 128, 128) 123008 ['max_pooling1d_1[0][0]']
dropout_9 (Dropout) (None, 128, 128) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 128) 0 ['dropout_9[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 16392 ['max_pooling1d_2[0][0]']
dropout_10 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_10[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 776 ['max_pooling1d_3[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
dropout_11 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_11[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
conv1d_5 (Conv1D) (None, 16, 8) 392 ['max_pooling1d_4[0][0]']
dense_5 (Dense) (None, 136) 18632 ['dropout_3[0][0]']
dropout_12 (Dropout) (None, 16, 8) 0 ['conv1d_5[0][0]']
dropout_4 (Dropout) (None, 136) 0 ['dense_5[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 8) 0 ['dropout_12[0][0]']
dense_6 (Dense) (None, 136) 18632 ['dropout_4[0][0]']
conv1d_6 (Conv1D) (None, 8, 8) 392 ['max_pooling1d_5[0][0]']
dropout_5 (Dropout) (None, 136) 0 ['dense_6[0][0]']
dropout_13 (Dropout) (None, 8, 8) 0 ['conv1d_6[0][0]']
dense_7 (Dense) (None, 136) 18632 ['dropout_5[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 8) 0 ['dropout_13[0][0]']
dropout_6 (Dropout) (None, 136) 0 ['dense_7[0][0]']
flatten_1 (Flatten) (None, 32) 0 ['max_pooling1d_6[0][0]']
dense_8 (Dense) (None, 8) 1096 ['dropout_6[0][0]']
dense_10 (Dense) (None, 128) 4224 ['flatten_1[0][0]']
dropout_7 (Dropout) (None, 8) 0 ['dense_8[0][0]']
dropout_14 (Dropout) (None, 128) 0 ['dense_10[0][0]']
concatenate (Concatenate) (None, 136) 0 ['dropout_7[0][0]',
'dropout_14[0][0]']
dense_12 (Dense) (None, 64) 8768 ['concatenate[0][0]']
dense_13 (Dense) (None, 1) 65 ['dense_12[0][0]']
==================================================================================================
Total params: 292,721
Trainable params: 292,721
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 18s 76ms/step - loss: 0.3083 - accuracy: 0.8943 - recall: 0.2452 - precision: 0.5811 - AUROC: 0.7362 - AUPRC: 0.3992 - f1_score: 0.3449 - balanced_accuracy: 0.6113 - specificity: 0.9774 - miss_rate: 0.7548 - fall_out: 0.0226 - mcc: 0.3307 - val_loss: 0.2771 - val_accuracy: 0.9057 - val_recall: 0.2021 - val_precision: 0.8580 - val_AUROC: 0.8522 - val_AUPRC: 0.5815 - val_f1_score: 0.3271 - val_balanced_accuracy: 0.5989 - val_specificity: 0.9957 - val_miss_rate: 0.7979 - val_fall_out: 0.0043 - val_mcc: 0.3890
Epoch 2/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2655 - accuracy: 0.9075 - recall: 0.3149 - precision: 0.7076 - AUROC: 0.8157 - AUPRC: 0.5227 - f1_score: 0.4358 - balanced_accuracy: 0.6491 - specificity: 0.9834 - miss_rate: 0.6851 - fall_out: 0.0166 - mcc: 0.4320 - val_loss: 0.2583 - val_accuracy: 0.9098 - val_recall: 0.2237 - val_precision: 0.9198 - val_AUROC: 0.8614 - val_AUPRC: 0.6048 - val_f1_score: 0.3599 - val_balanced_accuracy: 0.6106 - val_specificity: 0.9975 - val_miss_rate: 0.7763 - val_fall_out: 0.0025 - val_mcc: 0.4283
Epoch 3/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2543 - accuracy: 0.9104 - recall: 0.3312 - precision: 0.7321 - AUROC: 0.8346 - AUPRC: 0.5540 - f1_score: 0.4561 - balanced_accuracy: 0.6579 - specificity: 0.9845 - miss_rate: 0.6688 - fall_out: 0.0155 - mcc: 0.4538 - val_loss: 0.2484 - val_accuracy: 0.9149 - val_recall: 0.3052 - val_precision: 0.8456 - val_AUROC: 0.8634 - val_AUPRC: 0.6091 - val_f1_score: 0.4485 - val_balanced_accuracy: 0.6490 - val_specificity: 0.9929 - val_miss_rate: 0.6948 - val_fall_out: 0.0071 - val_mcc: 0.4770
Epoch 4/1000
198/198 [==============================] - 12s 59ms/step - loss: 0.2477 - accuracy: 0.9116 - recall: 0.3408 - precision: 0.7385 - AUROC: 0.8472 - AUPRC: 0.5722 - f1_score: 0.4664 - balanced_accuracy: 0.6627 - specificity: 0.9846 - miss_rate: 0.6592 - fall_out: 0.0154 - mcc: 0.4633 - val_loss: 0.2509 - val_accuracy: 0.9043 - val_recall: 0.1617 - val_precision: 0.9667 - val_AUROC: 0.8625 - val_AUPRC: 0.6092 - val_f1_score: 0.2770 - val_balanced_accuracy: 0.5805 - val_specificity: 0.9993 - val_miss_rate: 0.8383 - val_fall_out: 7.1289e-04 - val_mcc: 0.3742
Epoch 5/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2436 - accuracy: 0.9133 - recall: 0.3481 - precision: 0.7561 - AUROC: 0.8532 - AUPRC: 0.5843 - f1_score: 0.4767 - balanced_accuracy: 0.6669 - specificity: 0.9856 - miss_rate: 0.6519 - fall_out: 0.0144 - mcc: 0.4757 - val_loss: 0.2415 - val_accuracy: 0.9123 - val_recall: 0.2488 - val_precision: 0.9177 - val_AUROC: 0.8666 - val_AUPRC: 0.6161 - val_f1_score: 0.3914 - val_balanced_accuracy: 0.6230 - val_specificity: 0.9971 - val_miss_rate: 0.7512 - val_fall_out: 0.0029 - val_mcc: 0.4518
Epoch 6/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2401 - accuracy: 0.9138 - recall: 0.3514 - precision: 0.7598 - AUROC: 0.8594 - AUPRC: 0.5912 - f1_score: 0.4806 - balanced_accuracy: 0.6686 - specificity: 0.9858 - miss_rate: 0.6486 - fall_out: 0.0142 - mcc: 0.4796 - val_loss: 0.2423 - val_accuracy: 0.9139 - val_recall: 0.2753 - val_precision: 0.8876 - val_AUROC: 0.8643 - val_AUPRC: 0.6111 - val_f1_score: 0.4202 - val_balanced_accuracy: 0.6354 - val_specificity: 0.9955 - val_miss_rate: 0.7247 - val_fall_out: 0.0045 - val_mcc: 0.4662
Epoch 7/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2375 - accuracy: 0.9142 - recall: 0.3527 - precision: 0.7642 - AUROC: 0.8633 - AUPRC: 0.5965 - f1_score: 0.4826 - balanced_accuracy: 0.6694 - specificity: 0.9861 - miss_rate: 0.6473 - fall_out: 0.0139 - mcc: 0.4823 - val_loss: 0.2435 - val_accuracy: 0.9109 - val_recall: 0.2321 - val_precision: 0.9276 - val_AUROC: 0.8646 - val_AUPRC: 0.6120 - val_f1_score: 0.3712 - val_balanced_accuracy: 0.6149 - val_specificity: 0.9977 - val_miss_rate: 0.7679 - val_fall_out: 0.0023 - val_mcc: 0.4388
198/198 [==============================] - 7s 33ms/step - loss: 0.2284 - accuracy: 0.9110 - recall: 0.2396 - precision: 0.9071 - AUROC: 0.8946 - AUPRC: 0.6548 - f1_score: 0.3791 - balanced_accuracy: 0.6182 - specificity: 0.9969 - miss_rate: 0.7604 - fall_out: 0.0031 - mcc: 0.4399
50/50 [==============================] - 5s 92ms/step - loss: 0.2435 - accuracy: 0.9109 - recall: 0.2321 - precision: 0.9276 - AUROC: 0.8646 - AUPRC: 0.6120 - f1_score: 0.3712 - balanced_accuracy: 0.6149 - specificity: 0.9977 - miss_rate: 0.7679 - fall_out: 0.0023 - mcc: 0.4388
-- HOLDOUT 3 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['SNRNP70', 'ZNF282']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70', 'ZNF282']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 15s 63ms/step - loss: 17.1689 - accuracy: 0.8766 - recall: 0.2473 - precision: 0.4248 - AUROC: 0.7254 - AUPRC: 0.3433 - f1_score: 0.3126 - balanced_accuracy: 0.6022 - specificity: 0.9572 - miss_rate: 0.7527 - fall_out: 0.0428 - mcc: 0.2611 - val_loss: 2.3323 - val_accuracy: 0.8977 - val_recall: 0.1108 - val_precision: 0.8933 - val_AUROC: 0.8447 - val_AUPRC: 0.5781 - val_f1_score: 0.1971 - val_balanced_accuracy: 0.5546 - val_specificity: 0.9983 - val_miss_rate: 0.8892 - val_fall_out: 0.0017 - val_mcc: 0.2938
Epoch 2/1000
198/198 [==============================] - 13s 61ms/step - loss: 2.0686 - accuracy: 0.9060 - recall: 0.2936 - precision: 0.7054 - AUROC: 0.8196 - AUPRC: 0.5110 - f1_score: 0.4147 - balanced_accuracy: 0.6390 - specificity: 0.9843 - miss_rate: 0.7064 - fall_out: 0.0157 - mcc: 0.4156 - val_loss: 1.6999 - val_accuracy: 0.9061 - val_recall: 0.1993 - val_precision: 0.8800 - val_AUROC: 0.8587 - val_AUPRC: 0.6075 - val_f1_score: 0.3250 - val_balanced_accuracy: 0.5979 - val_specificity: 0.9965 - val_miss_rate: 0.8007 - val_fall_out: 0.0035 - val_mcc: 0.3925
Epoch 3/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.8197 - accuracy: 0.9081 - recall: 0.3081 - precision: 0.7217 - AUROC: 0.8356 - AUPRC: 0.5361 - f1_score: 0.4318 - balanced_accuracy: 0.6464 - specificity: 0.9848 - miss_rate: 0.6919 - fall_out: 0.0152 - mcc: 0.4327 - val_loss: 1.5652 - val_accuracy: 0.8979 - val_recall: 0.1038 - val_precision: 0.9613 - val_AUROC: 0.8645 - val_AUPRC: 0.6133 - val_f1_score: 0.1874 - val_balanced_accuracy: 0.5516 - val_specificity: 0.9995 - val_miss_rate: 0.8962 - val_fall_out: 5.3466e-04 - val_mcc: 0.2978
Epoch 4/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.6947 - accuracy: 0.9099 - recall: 0.3210 - precision: 0.7354 - AUROC: 0.8440 - AUPRC: 0.5547 - f1_score: 0.4469 - balanced_accuracy: 0.6531 - specificity: 0.9852 - miss_rate: 0.6790 - fall_out: 0.0148 - mcc: 0.4476 - val_loss: 1.4318 - val_accuracy: 0.9008 - val_recall: 0.1394 - val_precision: 0.9091 - val_AUROC: 0.8653 - val_AUPRC: 0.6115 - val_f1_score: 0.2417 - val_balanced_accuracy: 0.5688 - val_specificity: 0.9982 - val_miss_rate: 0.8606 - val_fall_out: 0.0018 - val_mcc: 0.3338
Epoch 5/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.6013 - accuracy: 0.9109 - recall: 0.3285 - precision: 0.7428 - AUROC: 0.8475 - AUPRC: 0.5622 - f1_score: 0.4555 - balanced_accuracy: 0.6570 - specificity: 0.9855 - miss_rate: 0.6715 - fall_out: 0.0145 - mcc: 0.4561 - val_loss: 1.3426 - val_accuracy: 0.9068 - val_recall: 0.1958 - val_precision: 0.9183 - val_AUROC: 0.8678 - val_AUPRC: 0.6231 - val_f1_score: 0.3228 - val_balanced_accuracy: 0.5968 - val_specificity: 0.9978 - val_miss_rate: 0.8042 - val_fall_out: 0.0022 - val_mcc: 0.3996
Epoch 6/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.5268 - accuracy: 0.9114 - recall: 0.3307 - precision: 0.7465 - AUROC: 0.8558 - AUPRC: 0.5694 - f1_score: 0.4584 - balanced_accuracy: 0.6582 - specificity: 0.9856 - miss_rate: 0.6693 - fall_out: 0.0144 - mcc: 0.4592 - val_loss: 1.2436 - val_accuracy: 0.9102 - val_recall: 0.2390 - val_precision: 0.8863 - val_AUROC: 0.8696 - val_AUPRC: 0.6239 - val_f1_score: 0.3765 - val_balanced_accuracy: 0.6176 - val_specificity: 0.9961 - val_miss_rate: 0.7610 - val_fall_out: 0.0039 - val_mcc: 0.4330
Epoch 7/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.4648 - accuracy: 0.9128 - recall: 0.3380 - precision: 0.7597 - AUROC: 0.8602 - AUPRC: 0.5815 - f1_score: 0.4679 - balanced_accuracy: 0.6622 - specificity: 0.9863 - miss_rate: 0.6620 - fall_out: 0.0137 - mcc: 0.4699 - val_loss: 1.2142 - val_accuracy: 0.9153 - val_recall: 0.3094 - val_precision: 0.8457 - val_AUROC: 0.8689 - val_AUPRC: 0.6275 - val_f1_score: 0.4531 - val_balanced_accuracy: 0.6511 - val_specificity: 0.9928 - val_miss_rate: 0.6906 - val_fall_out: 0.0072 - val_mcc: 0.4805
Epoch 8/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.4162 - accuracy: 0.9128 - recall: 0.3380 - precision: 0.7600 - AUROC: 0.8621 - AUPRC: 0.5856 - f1_score: 0.4679 - balanced_accuracy: 0.6622 - specificity: 0.9863 - miss_rate: 0.6620 - fall_out: 0.0137 - mcc: 0.4700 - val_loss: 1.1657 - val_accuracy: 0.9121 - val_recall: 0.2627 - val_precision: 0.8727 - val_AUROC: 0.8693 - val_AUPRC: 0.6280 - val_f1_score: 0.4039 - val_balanced_accuracy: 0.6289 - val_specificity: 0.9951 - val_miss_rate: 0.7373 - val_fall_out: 0.0049 - val_mcc: 0.4502
Epoch 9/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.3730 - accuracy: 0.9132 - recall: 0.3431 - precision: 0.7592 - AUROC: 0.8660 - AUPRC: 0.5910 - f1_score: 0.4726 - balanced_accuracy: 0.6646 - specificity: 0.9861 - miss_rate: 0.6569 - fall_out: 0.0139 - mcc: 0.4733 - val_loss: 1.1263 - val_accuracy: 0.9122 - val_recall: 0.2544 - val_precision: 0.8990 - val_AUROC: 0.8702 - val_AUPRC: 0.6307 - val_f1_score: 0.3965 - val_balanced_accuracy: 0.6254 - val_specificity: 0.9963 - val_miss_rate: 0.7456 - val_fall_out: 0.0037 - val_mcc: 0.4511
Epoch 10/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.3382 - accuracy: 0.9139 - recall: 0.3518 - precision: 0.7605 - AUROC: 0.8677 - AUPRC: 0.5950 - f1_score: 0.4811 - balanced_accuracy: 0.6688 - specificity: 0.9858 - miss_rate: 0.6482 - fall_out: 0.0142 - mcc: 0.4802 - val_loss: 1.1041 - val_accuracy: 0.9139 - val_recall: 0.2774 - val_precision: 0.8825 - val_AUROC: 0.8713 - val_AUPRC: 0.6290 - val_f1_score: 0.4221 - val_balanced_accuracy: 0.6363 - val_specificity: 0.9953 - val_miss_rate: 0.7226 - val_fall_out: 0.0047 - val_mcc: 0.4663
Epoch 11/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.3030 - accuracy: 0.9136 - recall: 0.3520 - precision: 0.7552 - AUROC: 0.8748 - AUPRC: 0.6036 - f1_score: 0.4802 - balanced_accuracy: 0.6687 - specificity: 0.9854 - miss_rate: 0.6480 - fall_out: 0.0146 - mcc: 0.4781 - val_loss: 1.0549 - val_accuracy: 0.9091 - val_recall: 0.2216 - val_precision: 0.9060 - val_AUROC: 0.8692 - val_AUPRC: 0.6267 - val_f1_score: 0.3561 - val_balanced_accuracy: 0.6093 - val_specificity: 0.9971 - val_miss_rate: 0.7784 - val_fall_out: 0.0029 - val_mcc: 0.4222
Epoch 12/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.2720 - accuracy: 0.9135 - recall: 0.3542 - precision: 0.7514 - AUROC: 0.8752 - AUPRC: 0.6108 - f1_score: 0.4815 - balanced_accuracy: 0.6696 - specificity: 0.9850 - miss_rate: 0.6458 - fall_out: 0.0150 - mcc: 0.4782 - val_loss: 1.0387 - val_accuracy: 0.9100 - val_recall: 0.2307 - val_precision: 0.9044 - val_AUROC: 0.8691 - val_AUPRC: 0.6280 - val_f1_score: 0.3676 - val_balanced_accuracy: 0.6138 - val_specificity: 0.9969 - val_miss_rate: 0.7693 - val_fall_out: 0.0031 - val_mcc: 0.4305
Epoch 13/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.2453 - accuracy: 0.9143 - recall: 0.3603 - precision: 0.7570 - AUROC: 0.8811 - AUPRC: 0.6115 - f1_score: 0.4883 - balanced_accuracy: 0.6728 - specificity: 0.9852 - miss_rate: 0.6397 - fall_out: 0.0148 - mcc: 0.4849 - val_loss: 1.0501 - val_accuracy: 0.9145 - val_recall: 0.2948 - val_precision: 0.8580 - val_AUROC: 0.8669 - val_AUPRC: 0.6297 - val_f1_score: 0.4388 - val_balanced_accuracy: 0.6443 - val_specificity: 0.9938 - val_miss_rate: 0.7052 - val_fall_out: 0.0062 - val_mcc: 0.4728
Epoch 14/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.2192 - accuracy: 0.9142 - recall: 0.3565 - precision: 0.7590 - AUROC: 0.8824 - AUPRC: 0.6181 - f1_score: 0.4851 - balanced_accuracy: 0.6710 - specificity: 0.9855 - miss_rate: 0.6435 - fall_out: 0.0145 - mcc: 0.4829 - val_loss: 1.0082 - val_accuracy: 0.9158 - val_recall: 0.3192 - val_precision: 0.8373 - val_AUROC: 0.8670 - val_AUPRC: 0.6262 - val_f1_score: 0.4622 - val_balanced_accuracy: 0.6556 - val_specificity: 0.9921 - val_miss_rate: 0.6808 - val_fall_out: 0.0079 - val_mcc: 0.4853
Epoch 15/1000
198/198 [==============================] - 13s 61ms/step - loss: 1.1934 - accuracy: 0.9153 - recall: 0.3755 - precision: 0.7544 - AUROC: 0.8865 - AUPRC: 0.6282 - f1_score: 0.5014 - balanced_accuracy: 0.6799 - specificity: 0.9844 - miss_rate: 0.6245 - fall_out: 0.0156 - mcc: 0.4944 - val_loss: 1.0307 - val_accuracy: 0.9144 - val_recall: 0.2955 - val_precision: 0.8531 - val_AUROC: 0.8665 - val_AUPRC: 0.6248 - val_f1_score: 0.4389 - val_balanced_accuracy: 0.6445 - val_specificity: 0.9935 - val_miss_rate: 0.7045 - val_fall_out: 0.0065 - val_mcc: 0.4717
Epoch 16/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.1695 - accuracy: 0.9166 - recall: 0.3905 - precision: 0.7562 - AUROC: 0.8915 - AUPRC: 0.6365 - f1_score: 0.5150 - balanced_accuracy: 0.6872 - specificity: 0.9839 - miss_rate: 0.6095 - fall_out: 0.0161 - mcc: 0.5055 - val_loss: 1.0615 - val_accuracy: 0.9140 - val_recall: 0.2913 - val_precision: 0.8531 - val_AUROC: 0.8656 - val_AUPRC: 0.6258 - val_f1_score: 0.4343 - val_balanced_accuracy: 0.6424 - val_specificity: 0.9936 - val_miss_rate: 0.7087 - val_fall_out: 0.0064 - val_mcc: 0.4682
198/198 [==============================] - 7s 34ms/step - loss: 1.0350 - accuracy: 0.9139 - recall: 0.2774 - precision: 0.8830 - AUROC: 0.9181 - AUPRC: 0.6828 - f1_score: 0.4222 - balanced_accuracy: 0.6364 - specificity: 0.9953 - miss_rate: 0.7226 - fall_out: 0.0047 - mcc: 0.4665
50/50 [==============================] - 5s 90ms/step - loss: 1.0615 - accuracy: 0.9140 - recall: 0.2913 - precision: 0.8531 - AUROC: 0.8656 - AUPRC: 0.6258 - f1_score: 0.4343 - balanced_accuracy: 0.6424 - specificity: 0.9936 - miss_rate: 0.7087 - fall_out: 0.0064 - mcc: 0.4682
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_2\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_2\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 3, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 136, 'dropout_1': 0.5, 'units_2': 8, 'dropout_2': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_2\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_2\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 6, 'conv_dropout_0': 0.2, 'dense_depth': 1, 'units_0': 16, 'dense_dropout_0': 0.3, 'filters_1': 8, 'kernel_size_1': 6, 'conv_dropout_1': 0.0, 'filters_2': 8, 'kernel_size_2': 12, 'conv_dropout_2': 0.0, 'filters_3': 120, 'kernel_size_3': 8, 'conv_dropout_3': 0.0, 'filters_4': 160, 'kernel_size_4': 6, 'conv_dropout_4': 0.0, 'units_1': 16, 'dense_dropout_1': 0.3, 'units_2': 80, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.3, 'filters_5': 8, 'kernel_size_5': 16, 'conv_dropout_5': 0.0, 'units_4': 16, 'dense_dropout_4': 0.3}
- Removing uncorrelated/correlated features: ['SNRNP70', 'ZNF282']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 136) 18632
dropout_4 (Dropout) (None, 136) 0
dense_6 (Dense) (None, 8) 1096
dropout_5 (Dropout) (None, 8) 0
dense_7 (Dense) (None, 1) 9
=================================================================
Total params: 96,169
Trainable params: 96,169
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 4000
dropout_6 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 8) 7688
dropout_7 (Dropout) (None, 128, 8) 0
max_pooling1d_2 (MaxPooling (None, 64, 8) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 776
dropout_8 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
conv1d_4 (Conv1D) (None, 32, 120) 7800
dropout_9 (Dropout) (None, 32, 120) 0
max_pooling1d_4 (MaxPooling (None, 16, 120) 0
1D)
conv1d_5 (Conv1D) (None, 16, 160) 115360
dropout_10 (Dropout) (None, 16, 160) 0
max_pooling1d_5 (MaxPooling (None, 8, 160) 0
1D)
conv1d_6 (Conv1D) (None, 8, 8) 20488
dropout_11 (Dropout) (None, 8, 8) 0
max_pooling1d_6 (MaxPooling (None, 4, 8) 0
1D)
flatten_1 (Flatten) (None, 32) 0
dense_8 (Dense) (None, 16) 528
dropout_12 (Dropout) (None, 16) 0
dense_9 (Dense) (None, 1) 17
=================================================================
Total params: 156,657
Trainable params: 156,657
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 4000 ['sequence_data[0][0]']
dropout_6 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_6[0][0]']
conv1d_2 (Conv1D) (None, 128, 8) 7688 ['max_pooling1d_1[0][0]']
dropout_7 (Dropout) (None, 128, 8) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 8) 0 ['dropout_7[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 776 ['max_pooling1d_2[0][0]']
dropout_8 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_8[0][0]']
conv1d_4 (Conv1D) (None, 32, 120) 7800 ['max_pooling1d_3[0][0]']
dropout_9 (Dropout) (None, 32, 120) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 120) 0 ['dropout_9[0][0]']
conv1d_5 (Conv1D) (None, 16, 160) 115360 ['max_pooling1d_4[0][0]']
dropout_10 (Dropout) (None, 16, 160) 0 ['conv1d_5[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_5 (MaxPooling1D) (None, 8, 160) 0 ['dropout_10[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
conv1d_6 (Conv1D) (None, 8, 8) 20488 ['max_pooling1d_5[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_11 (Dropout) (None, 8, 8) 0 ['conv1d_6[0][0]']
dense_5 (Dense) (None, 136) 18632 ['dropout_3[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 8) 0 ['dropout_11[0][0]']
dropout_4 (Dropout) (None, 136) 0 ['dense_5[0][0]']
flatten_1 (Flatten) (None, 32) 0 ['max_pooling1d_6[0][0]']
dense_6 (Dense) (None, 8) 1096 ['dropout_4[0][0]']
dense_8 (Dense) (None, 16) 528 ['flatten_1[0][0]']
dropout_5 (Dropout) (None, 8) 0 ['dense_6[0][0]']
dropout_12 (Dropout) (None, 16) 0 ['dense_8[0][0]']
concatenate (Concatenate) (None, 24) 0 ['dropout_5[0][0]',
'dropout_12[0][0]']
dense_10 (Dense) (None, 64) 1600 ['concatenate[0][0]']
dense_11 (Dense) (None, 1) 65 ['dense_10[0][0]']
==================================================================================================
Total params: 254,465
Trainable params: 254,465
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 16s 59ms/step - loss: 0.3066 - accuracy: 0.8983 - recall: 0.2463 - precision: 0.6321 - AUROC: 0.7504 - AUPRC: 0.4086 - f1_score: 0.3544 - balanced_accuracy: 0.6140 - specificity: 0.9817 - miss_rate: 0.7537 - fall_out: 0.0183 - mcc: 0.3517 - val_loss: 0.2495 - val_accuracy: 0.9157 - val_recall: 0.4314 - val_precision: 0.7115 - val_AUROC: 0.8519 - val_AUPRC: 0.5989 - val_f1_score: 0.5371 - val_balanced_accuracy: 0.7045 - val_specificity: 0.9776 - val_miss_rate: 0.5686 - val_fall_out: 0.0224 - val_mcc: 0.5125
Epoch 2/1000
198/198 [==============================] - 11s 53ms/step - loss: 0.2654 - accuracy: 0.9081 - recall: 0.3279 - precision: 0.7029 - AUROC: 0.8195 - AUPRC: 0.5249 - f1_score: 0.4472 - balanced_accuracy: 0.6551 - specificity: 0.9823 - miss_rate: 0.6721 - fall_out: 0.0177 - mcc: 0.4394 - val_loss: 0.2430 - val_accuracy: 0.9091 - val_recall: 0.2230 - val_precision: 0.8989 - val_AUROC: 0.8639 - val_AUPRC: 0.6143 - val_f1_score: 0.3573 - val_balanced_accuracy: 0.6099 - val_specificity: 0.9968 - val_miss_rate: 0.7770 - val_fall_out: 0.0032 - val_mcc: 0.4215
Epoch 3/1000
198/198 [==============================] - 11s 54ms/step - loss: 0.2554 - accuracy: 0.9101 - recall: 0.3307 - precision: 0.7284 - AUROC: 0.8392 - AUPRC: 0.5497 - f1_score: 0.4549 - balanced_accuracy: 0.6575 - specificity: 0.9842 - miss_rate: 0.6693 - fall_out: 0.0158 - mcc: 0.4519 - val_loss: 0.2487 - val_accuracy: 0.9108 - val_recall: 0.2453 - val_precision: 0.8844 - val_AUROC: 0.8668 - val_AUPRC: 0.6204 - val_f1_score: 0.3841 - val_balanced_accuracy: 0.6206 - val_specificity: 0.9959 - val_miss_rate: 0.7547 - val_fall_out: 0.0041 - val_mcc: 0.4382
Epoch 4/1000
198/198 [==============================] - 11s 54ms/step - loss: 0.2528 - accuracy: 0.9109 - recall: 0.3290 - precision: 0.7414 - AUROC: 0.8446 - AUPRC: 0.5549 - f1_score: 0.4557 - balanced_accuracy: 0.6571 - specificity: 0.9853 - miss_rate: 0.6710 - fall_out: 0.0147 - mcc: 0.4559 - val_loss: 0.2436 - val_accuracy: 0.9142 - val_recall: 0.2864 - val_precision: 0.8689 - val_AUROC: 0.8708 - val_AUPRC: 0.6272 - val_f1_score: 0.4308 - val_balanced_accuracy: 0.6404 - val_specificity: 0.9945 - val_miss_rate: 0.7136 - val_fall_out: 0.0055 - val_mcc: 0.4695
198/198 [==============================] - 7s 34ms/step - loss: 0.2437 - accuracy: 0.9127 - recall: 0.2726 - precision: 0.8656 - AUROC: 0.8756 - AUPRC: 0.6167 - f1_score: 0.4146 - balanced_accuracy: 0.6336 - specificity: 0.9946 - miss_rate: 0.7274 - fall_out: 0.0054 - mcc: 0.4565
50/50 [==============================] - 5s 95ms/step - loss: 0.2436 - accuracy: 0.9142 - recall: 0.2864 - precision: 0.8689 - AUROC: 0.8708 - AUPRC: 0.6272 - f1_score: 0.4308 - balanced_accuracy: 0.6404 - specificity: 0.9945 - miss_rate: 0.7136 - fall_out: 0.0055 - mcc: 0.4695
-- HOLDOUT 4 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['SNRNP70', 'ATM']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70', 'ATM']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 15s 64ms/step - loss: 17.4880 - accuracy: 0.8817 - recall: 0.2294 - precision: 0.4575 - AUROC: 0.7290 - AUPRC: 0.3450 - f1_score: 0.3055 - balanced_accuracy: 0.5973 - specificity: 0.9652 - miss_rate: 0.7706 - fall_out: 0.0348 - mcc: 0.2664 - val_loss: 2.5179 - val_accuracy: 0.9093 - val_recall: 0.2383 - val_precision: 0.8615 - val_AUROC: 0.8502 - val_AUPRC: 0.5855 - val_f1_score: 0.3734 - val_balanced_accuracy: 0.6167 - val_specificity: 0.9951 - val_miss_rate: 0.7617 - val_fall_out: 0.0049 - val_mcc: 0.4246
Epoch 2/1000
198/198 [==============================] - 13s 62ms/step - loss: 2.0432 - accuracy: 0.9073 - recall: 0.3002 - precision: 0.7180 - AUROC: 0.8252 - AUPRC: 0.5254 - f1_score: 0.4234 - balanced_accuracy: 0.6426 - specificity: 0.9849 - miss_rate: 0.6998 - fall_out: 0.0151 - mcc: 0.4254 - val_loss: 2.0486 - val_accuracy: 0.9057 - val_recall: 0.1951 - val_precision: 0.8805 - val_AUROC: 0.8605 - val_AUPRC: 0.5934 - val_f1_score: 0.3195 - val_balanced_accuracy: 0.5959 - val_specificity: 0.9966 - val_miss_rate: 0.8049 - val_fall_out: 0.0034 - val_mcc: 0.3884
Epoch 3/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.8093 - accuracy: 0.9095 - recall: 0.3208 - precision: 0.7304 - AUROC: 0.8362 - AUPRC: 0.5435 - f1_score: 0.4458 - balanced_accuracy: 0.6528 - specificity: 0.9849 - miss_rate: 0.6792 - fall_out: 0.0151 - mcc: 0.4455 - val_loss: 1.9541 - val_accuracy: 0.8888 - val_recall: 0.0223 - val_precision: 0.8649 - val_AUROC: 0.8645 - val_AUPRC: 0.5765 - val_f1_score: 0.0435 - val_balanced_accuracy: 0.5109 - val_specificity: 0.9996 - val_miss_rate: 0.9777 - val_fall_out: 4.4555e-04 - val_mcc: 0.1283
Epoch 4/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.6874 - accuracy: 0.9103 - recall: 0.3265 - precision: 0.7350 - AUROC: 0.8457 - AUPRC: 0.5600 - f1_score: 0.4522 - balanced_accuracy: 0.6557 - specificity: 0.9849 - miss_rate: 0.6735 - fall_out: 0.0151 - mcc: 0.4515 - val_loss: 1.8208 - val_accuracy: 0.9128 - val_recall: 0.2871 - val_precision: 0.8357 - val_AUROC: 0.8676 - val_AUPRC: 0.6034 - val_f1_score: 0.4274 - val_balanced_accuracy: 0.6399 - val_specificity: 0.9928 - val_miss_rate: 0.7129 - val_fall_out: 0.0072 - val_mcc: 0.4587
Epoch 5/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.5968 - accuracy: 0.9119 - recall: 0.3326 - precision: 0.7520 - AUROC: 0.8511 - AUPRC: 0.5703 - f1_score: 0.4612 - balanced_accuracy: 0.6593 - specificity: 0.9860 - miss_rate: 0.6674 - fall_out: 0.0140 - mcc: 0.4628 - val_loss: 1.7654 - val_accuracy: 0.9101 - val_recall: 0.2460 - val_precision: 0.8631 - val_AUROC: 0.8659 - val_AUPRC: 0.6029 - val_f1_score: 0.3829 - val_balanced_accuracy: 0.6205 - val_specificity: 0.9950 - val_miss_rate: 0.7540 - val_fall_out: 0.0050 - val_mcc: 0.4321
Epoch 6/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.5244 - accuracy: 0.9122 - recall: 0.3448 - precision: 0.7438 - AUROC: 0.8547 - AUPRC: 0.5772 - f1_score: 0.4712 - balanced_accuracy: 0.6648 - specificity: 0.9848 - miss_rate: 0.6552 - fall_out: 0.0152 - mcc: 0.4683 - val_loss: 1.6781 - val_accuracy: 0.9086 - val_recall: 0.2195 - val_precision: 0.8949 - val_AUROC: 0.8634 - val_AUPRC: 0.6015 - val_f1_score: 0.3525 - val_balanced_accuracy: 0.6081 - val_specificity: 0.9967 - val_miss_rate: 0.7805 - val_fall_out: 0.0033 - val_mcc: 0.4169
Epoch 7/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.4633 - accuracy: 0.9137 - recall: 0.3523 - precision: 0.7565 - AUROC: 0.8609 - AUPRC: 0.5881 - f1_score: 0.4808 - balanced_accuracy: 0.6689 - specificity: 0.9855 - miss_rate: 0.6477 - fall_out: 0.0145 - mcc: 0.4789 - val_loss: 1.6114 - val_accuracy: 0.9110 - val_recall: 0.2523 - val_precision: 0.8723 - val_AUROC: 0.8666 - val_AUPRC: 0.6087 - val_f1_score: 0.3914 - val_balanced_accuracy: 0.6238 - val_specificity: 0.9953 - val_miss_rate: 0.7477 - val_fall_out: 0.0047 - val_mcc: 0.4407
Epoch 8/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.4140 - accuracy: 0.9130 - recall: 0.3506 - precision: 0.7492 - AUROC: 0.8649 - AUPRC: 0.5947 - f1_score: 0.4776 - balanced_accuracy: 0.6678 - specificity: 0.9850 - miss_rate: 0.6494 - fall_out: 0.0150 - mcc: 0.4746 - val_loss: 1.5908 - val_accuracy: 0.9111 - val_recall: 0.2481 - val_precision: 0.8856 - val_AUROC: 0.8648 - val_AUPRC: 0.6071 - val_f1_score: 0.3876 - val_balanced_accuracy: 0.6220 - val_specificity: 0.9959 - val_miss_rate: 0.7519 - val_fall_out: 0.0041 - val_mcc: 0.4411
Epoch 9/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.3731 - accuracy: 0.9135 - recall: 0.3523 - precision: 0.7543 - AUROC: 0.8653 - AUPRC: 0.5948 - f1_score: 0.4803 - balanced_accuracy: 0.6688 - specificity: 0.9853 - miss_rate: 0.6477 - fall_out: 0.0147 - mcc: 0.4780 - val_loss: 1.5271 - val_accuracy: 0.9145 - val_recall: 0.3010 - val_precision: 0.8454 - val_AUROC: 0.8667 - val_AUPRC: 0.6092 - val_f1_score: 0.4440 - val_balanced_accuracy: 0.6470 - val_specificity: 0.9930 - val_miss_rate: 0.6990 - val_fall_out: 0.0070 - val_mcc: 0.4736
Epoch 10/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.3355 - accuracy: 0.9144 - recall: 0.3596 - precision: 0.7589 - AUROC: 0.8696 - AUPRC: 0.6007 - f1_score: 0.4880 - balanced_accuracy: 0.6725 - specificity: 0.9854 - miss_rate: 0.6404 - fall_out: 0.0146 - mcc: 0.4851 - val_loss: 1.5255 - val_accuracy: 0.9106 - val_recall: 0.2460 - val_precision: 0.8781 - val_AUROC: 0.8658 - val_AUPRC: 0.6093 - val_f1_score: 0.3843 - val_balanced_accuracy: 0.6208 - val_specificity: 0.9956 - val_miss_rate: 0.7540 - val_fall_out: 0.0044 - val_mcc: 0.4369
Epoch 11/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.3021 - accuracy: 0.9148 - recall: 0.3649 - precision: 0.7591 - AUROC: 0.8738 - AUPRC: 0.6086 - f1_score: 0.4928 - balanced_accuracy: 0.6750 - specificity: 0.9852 - miss_rate: 0.6351 - fall_out: 0.0148 - mcc: 0.4889 - val_loss: 1.5106 - val_accuracy: 0.9141 - val_recall: 0.2920 - val_precision: 0.8551 - val_AUROC: 0.8648 - val_AUPRC: 0.6115 - val_f1_score: 0.4353 - val_balanced_accuracy: 0.6428 - val_specificity: 0.9937 - val_miss_rate: 0.7080 - val_fall_out: 0.0063 - val_mcc: 0.4695
Epoch 12/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.2714 - accuracy: 0.9152 - recall: 0.3730 - precision: 0.7561 - AUROC: 0.8787 - AUPRC: 0.6145 - f1_score: 0.4996 - balanced_accuracy: 0.6788 - specificity: 0.9846 - miss_rate: 0.6270 - fall_out: 0.0154 - mcc: 0.4934 - val_loss: 1.4430 - val_accuracy: 0.9155 - val_recall: 0.3220 - val_precision: 0.8280 - val_AUROC: 0.8664 - val_AUPRC: 0.6144 - val_f1_score: 0.4636 - val_balanced_accuracy: 0.6567 - val_specificity: 0.9914 - val_miss_rate: 0.6780 - val_fall_out: 0.0086 - val_mcc: 0.4840
Epoch 13/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.2423 - accuracy: 0.9161 - recall: 0.3837 - precision: 0.7563 - AUROC: 0.8824 - AUPRC: 0.6239 - f1_score: 0.5091 - balanced_accuracy: 0.6839 - specificity: 0.9842 - miss_rate: 0.6163 - fall_out: 0.0158 - mcc: 0.5009 - val_loss: 1.4417 - val_accuracy: 0.9161 - val_recall: 0.3317 - val_precision: 0.8221 - val_AUROC: 0.8645 - val_AUPRC: 0.6102 - val_f1_score: 0.4727 - val_balanced_accuracy: 0.6613 - val_specificity: 0.9908 - val_miss_rate: 0.6683 - val_fall_out: 0.0092 - val_mcc: 0.4894
Epoch 14/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.2163 - accuracy: 0.9156 - recall: 0.3866 - precision: 0.7477 - AUROC: 0.8849 - AUPRC: 0.6312 - f1_score: 0.5097 - balanced_accuracy: 0.6850 - specificity: 0.9833 - miss_rate: 0.6134 - fall_out: 0.0167 - mcc: 0.4993 - val_loss: 1.3712 - val_accuracy: 0.9166 - val_recall: 0.3652 - val_precision: 0.7844 - val_AUROC: 0.8632 - val_AUPRC: 0.6110 - val_f1_score: 0.4983 - val_balanced_accuracy: 0.6762 - val_specificity: 0.9872 - val_miss_rate: 0.6348 - val_fall_out: 0.0128 - val_mcc: 0.4996
Epoch 15/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.1915 - accuracy: 0.9167 - recall: 0.3966 - precision: 0.7512 - AUROC: 0.8875 - AUPRC: 0.6383 - f1_score: 0.5191 - balanced_accuracy: 0.6899 - specificity: 0.9832 - miss_rate: 0.6034 - fall_out: 0.0168 - mcc: 0.5076 - val_loss: 1.3310 - val_accuracy: 0.9150 - val_recall: 0.3164 - val_precision: 0.8270 - val_AUROC: 0.8631 - val_AUPRC: 0.6098 - val_f1_score: 0.4577 - val_balanced_accuracy: 0.6540 - val_specificity: 0.9915 - val_miss_rate: 0.6836 - val_fall_out: 0.0085 - val_mcc: 0.4793
Epoch 16/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.1694 - accuracy: 0.9168 - recall: 0.4068 - precision: 0.7439 - AUROC: 0.8903 - AUPRC: 0.6377 - f1_score: 0.5260 - balanced_accuracy: 0.6945 - specificity: 0.9821 - miss_rate: 0.5932 - fall_out: 0.0179 - mcc: 0.5113 - val_loss: 1.2968 - val_accuracy: 0.9138 - val_recall: 0.3066 - val_precision: 0.8209 - val_AUROC: 0.8622 - val_AUPRC: 0.6005 - val_f1_score: 0.4465 - val_balanced_accuracy: 0.6490 - val_specificity: 0.9914 - val_miss_rate: 0.6934 - val_fall_out: 0.0086 - val_mcc: 0.4693
Epoch 17/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.1485 - accuracy: 0.9169 - recall: 0.4119 - precision: 0.7402 - AUROC: 0.8916 - AUPRC: 0.6417 - f1_score: 0.5293 - balanced_accuracy: 0.6967 - specificity: 0.9815 - miss_rate: 0.5881 - fall_out: 0.0185 - mcc: 0.5130 - val_loss: 1.2770 - val_accuracy: 0.9166 - val_recall: 0.3484 - val_precision: 0.8052 - val_AUROC: 0.8617 - val_AUPRC: 0.6092 - val_f1_score: 0.4864 - val_balanced_accuracy: 0.6688 - val_specificity: 0.9892 - val_miss_rate: 0.6516 - val_fall_out: 0.0108 - val_mcc: 0.4956
Epoch 18/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.1259 - accuracy: 0.9176 - recall: 0.4235 - precision: 0.7388 - AUROC: 0.8967 - AUPRC: 0.6504 - f1_score: 0.5384 - balanced_accuracy: 0.7022 - specificity: 0.9808 - miss_rate: 0.5765 - fall_out: 0.0192 - mcc: 0.5201 - val_loss: 1.2021 - val_accuracy: 0.9167 - val_recall: 0.3777 - val_precision: 0.7710 - val_AUROC: 0.8610 - val_AUPRC: 0.6082 - val_f1_score: 0.5070 - val_balanced_accuracy: 0.6817 - val_specificity: 0.9857 - val_miss_rate: 0.6223 - val_fall_out: 0.0143 - val_mcc: 0.5030
Epoch 19/1000
198/198 [==============================] - 13s 62ms/step - loss: 1.1026 - accuracy: 0.9181 - recall: 0.4406 - precision: 0.7302 - AUROC: 0.9009 - AUPRC: 0.6614 - f1_score: 0.5496 - balanced_accuracy: 0.7099 - specificity: 0.9792 - miss_rate: 0.5594 - fall_out: 0.0208 - mcc: 0.5272 - val_loss: 1.1649 - val_accuracy: 0.9161 - val_recall: 0.3909 - val_precision: 0.7490 - val_AUROC: 0.8573 - val_AUPRC: 0.6062 - val_f1_score: 0.5137 - val_balanced_accuracy: 0.6871 - val_specificity: 0.9832 - val_miss_rate: 0.6091 - val_fall_out: 0.0168 - val_mcc: 0.5028
Epoch 20/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.0839 - accuracy: 0.9196 - recall: 0.4545 - precision: 0.7350 - AUROC: 0.9018 - AUPRC: 0.6647 - f1_score: 0.5617 - balanced_accuracy: 0.7168 - specificity: 0.9790 - miss_rate: 0.5455 - fall_out: 0.0210 - mcc: 0.5384 - val_loss: 1.1587 - val_accuracy: 0.9165 - val_recall: 0.3512 - val_precision: 0.8000 - val_AUROC: 0.8571 - val_AUPRC: 0.6055 - val_f1_score: 0.4881 - val_balanced_accuracy: 0.6700 - val_specificity: 0.9888 - val_miss_rate: 0.6488 - val_fall_out: 0.0112 - val_mcc: 0.4957
Epoch 21/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.0636 - accuracy: 0.9188 - recall: 0.4657 - precision: 0.7194 - AUROC: 0.9062 - AUPRC: 0.6706 - f1_score: 0.5654 - balanced_accuracy: 0.7212 - specificity: 0.9768 - miss_rate: 0.5343 - fall_out: 0.0232 - mcc: 0.5379 - val_loss: 1.1366 - val_accuracy: 0.9164 - val_recall: 0.3819 - val_precision: 0.7622 - val_AUROC: 0.8557 - val_AUPRC: 0.6034 - val_f1_score: 0.5088 - val_balanced_accuracy: 0.6833 - val_specificity: 0.9848 - val_miss_rate: 0.6181 - val_fall_out: 0.0152 - val_mcc: 0.5022
Epoch 22/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.0455 - accuracy: 0.9210 - recall: 0.4800 - precision: 0.7308 - AUROC: 0.9080 - AUPRC: 0.6750 - f1_score: 0.5794 - balanced_accuracy: 0.7287 - specificity: 0.9774 - miss_rate: 0.5200 - fall_out: 0.0226 - mcc: 0.5524 - val_loss: 1.1183 - val_accuracy: 0.9164 - val_recall: 0.3610 - val_precision: 0.7860 - val_AUROC: 0.8558 - val_AUPRC: 0.6044 - val_f1_score: 0.4947 - val_balanced_accuracy: 0.6742 - val_specificity: 0.9874 - val_miss_rate: 0.6390 - val_fall_out: 0.0126 - val_mcc: 0.4972
Epoch 23/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.0279 - accuracy: 0.9208 - recall: 0.4934 - precision: 0.7201 - AUROC: 0.9106 - AUPRC: 0.6805 - f1_score: 0.5856 - balanced_accuracy: 0.7344 - specificity: 0.9755 - miss_rate: 0.5066 - fall_out: 0.0245 - mcc: 0.5554 - val_loss: 1.0907 - val_accuracy: 0.9173 - val_recall: 0.4000 - val_precision: 0.7553 - val_AUROC: 0.8449 - val_AUPRC: 0.6011 - val_f1_score: 0.5230 - val_balanced_accuracy: 0.6917 - val_specificity: 0.9834 - val_miss_rate: 0.6000 - val_fall_out: 0.0166 - val_mcc: 0.5117
Epoch 24/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.0089 - accuracy: 0.9215 - recall: 0.5125 - precision: 0.7150 - AUROC: 0.9139 - AUPRC: 0.6866 - f1_score: 0.5971 - balanced_accuracy: 0.7432 - specificity: 0.9739 - miss_rate: 0.4875 - fall_out: 0.0261 - mcc: 0.5644 - val_loss: 1.0784 - val_accuracy: 0.9161 - val_recall: 0.3847 - val_precision: 0.7551 - val_AUROC: 0.8491 - val_AUPRC: 0.5970 - val_f1_score: 0.5097 - val_balanced_accuracy: 0.6844 - val_specificity: 0.9840 - val_miss_rate: 0.6153 - val_fall_out: 0.0160 - val_mcc: 0.5011
Epoch 25/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.9907 - accuracy: 0.9241 - recall: 0.5350 - precision: 0.7233 - AUROC: 0.9170 - AUPRC: 0.6981 - f1_score: 0.6151 - balanced_accuracy: 0.7544 - specificity: 0.9738 - miss_rate: 0.4650 - fall_out: 0.0262 - mcc: 0.5820 - val_loss: 1.0236 - val_accuracy: 0.9161 - val_recall: 0.4195 - val_precision: 0.7244 - val_AUROC: 0.8531 - val_AUPRC: 0.6013 - val_f1_score: 0.5313 - val_balanced_accuracy: 0.6996 - val_specificity: 0.9796 - val_miss_rate: 0.5805 - val_fall_out: 0.0204 - val_mcc: 0.5109
Epoch 26/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.9733 - accuracy: 0.9251 - recall: 0.5489 - precision: 0.7241 - AUROC: 0.9202 - AUPRC: 0.7061 - f1_score: 0.6245 - balanced_accuracy: 0.7611 - specificity: 0.9732 - miss_rate: 0.4511 - fall_out: 0.0268 - mcc: 0.5907 - val_loss: 1.0285 - val_accuracy: 0.9159 - val_recall: 0.3951 - val_precision: 0.7421 - val_AUROC: 0.8508 - val_AUPRC: 0.5988 - val_f1_score: 0.5157 - val_balanced_accuracy: 0.6888 - val_specificity: 0.9824 - val_miss_rate: 0.6049 - val_fall_out: 0.0176 - val_mcc: 0.5026
Epoch 27/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.9569 - accuracy: 0.9261 - recall: 0.5524 - precision: 0.7302 - AUROC: 0.9228 - AUPRC: 0.7083 - f1_score: 0.6290 - balanced_accuracy: 0.7632 - specificity: 0.9739 - miss_rate: 0.4476 - fall_out: 0.0261 - mcc: 0.5959 - val_loss: 1.0166 - val_accuracy: 0.9154 - val_recall: 0.3986 - val_precision: 0.7333 - val_AUROC: 0.8474 - val_AUPRC: 0.5972 - val_f1_score: 0.5165 - val_balanced_accuracy: 0.6900 - val_specificity: 0.9815 - val_miss_rate: 0.6014 - val_fall_out: 0.0185 - val_mcc: 0.5011
Epoch 28/1000
198/198 [==============================] - 13s 63ms/step - loss: 0.9410 - accuracy: 0.9285 - recall: 0.5648 - precision: 0.7430 - AUROC: 0.9234 - AUPRC: 0.7164 - f1_score: 0.6417 - balanced_accuracy: 0.7699 - specificity: 0.9750 - miss_rate: 0.4352 - fall_out: 0.0250 - mcc: 0.6098 - val_loss: 1.0012 - val_accuracy: 0.9120 - val_recall: 0.4509 - val_precision: 0.6650 - val_AUROC: 0.8520 - val_AUPRC: 0.6001 - val_f1_score: 0.5374 - val_balanced_accuracy: 0.7109 - val_specificity: 0.9709 - val_miss_rate: 0.5491 - val_fall_out: 0.0291 - val_mcc: 0.5020
Epoch 29/1000
198/198 [==============================] - 13s 63ms/step - loss: 0.9237 - accuracy: 0.9282 - recall: 0.5754 - precision: 0.7341 - AUROC: 0.9269 - AUPRC: 0.7255 - f1_score: 0.6451 - balanced_accuracy: 0.7744 - specificity: 0.9733 - miss_rate: 0.4246 - fall_out: 0.0267 - mcc: 0.6114 - val_loss: 1.0088 - val_accuracy: 0.9148 - val_recall: 0.3986 - val_precision: 0.7268 - val_AUROC: 0.8479 - val_AUPRC: 0.5920 - val_f1_score: 0.5149 - val_balanced_accuracy: 0.6897 - val_specificity: 0.9808 - val_miss_rate: 0.6014 - val_fall_out: 0.0192 - val_mcc: 0.4982
Epoch 30/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.9090 - accuracy: 0.9291 - recall: 0.5808 - precision: 0.7380 - AUROC: 0.9287 - AUPRC: 0.7345 - f1_score: 0.6500 - balanced_accuracy: 0.7772 - specificity: 0.9736 - miss_rate: 0.4192 - fall_out: 0.0264 - mcc: 0.6166 - val_loss: 0.9870 - val_accuracy: 0.9144 - val_recall: 0.4125 - val_precision: 0.7107 - val_AUROC: 0.8470 - val_AUPRC: 0.5937 - val_f1_score: 0.5220 - val_balanced_accuracy: 0.6955 - val_specificity: 0.9785 - val_miss_rate: 0.5875 - val_fall_out: 0.0215 - val_mcc: 0.5000
Epoch 31/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.8970 - accuracy: 0.9301 - recall: 0.5897 - precision: 0.7408 - AUROC: 0.9292 - AUPRC: 0.7322 - f1_score: 0.6566 - balanced_accuracy: 0.7816 - specificity: 0.9736 - miss_rate: 0.4103 - fall_out: 0.0264 - mcc: 0.6233 - val_loss: 0.9135 - val_accuracy: 0.9126 - val_recall: 0.4411 - val_precision: 0.6756 - val_AUROC: 0.8492 - val_AUPRC: 0.5949 - val_f1_score: 0.5337 - val_balanced_accuracy: 0.7070 - val_specificity: 0.9729 - val_miss_rate: 0.5589 - val_fall_out: 0.0271 - val_mcc: 0.5014
Epoch 32/1000
198/198 [==============================] - 13s 62ms/step - loss: 0.8823 - accuracy: 0.9309 - recall: 0.6050 - precision: 0.7388 - AUROC: 0.9320 - AUPRC: 0.7412 - f1_score: 0.6653 - balanced_accuracy: 0.7888 - specificity: 0.9726 - miss_rate: 0.3950 - fall_out: 0.0274 - mcc: 0.6311 - val_loss: 0.9492 - val_accuracy: 0.9132 - val_recall: 0.3645 - val_precision: 0.7366 - val_AUROC: 0.8359 - val_AUPRC: 0.5738 - val_f1_score: 0.4876 - val_balanced_accuracy: 0.6739 - val_specificity: 0.9833 - val_miss_rate: 0.6355 - val_fall_out: 0.0167 - val_mcc: 0.4792
Epoch 33/1000
198/198 [==============================] - 13s 63ms/step - loss: 0.8697 - accuracy: 0.9307 - recall: 0.5937 - precision: 0.7435 - AUROC: 0.9331 - AUPRC: 0.7425 - f1_score: 0.6602 - balanced_accuracy: 0.7837 - specificity: 0.9738 - miss_rate: 0.4063 - fall_out: 0.0262 - mcc: 0.6270 - val_loss: 0.9837 - val_accuracy: 0.9132 - val_recall: 0.3889 - val_precision: 0.7163 - val_AUROC: 0.8391 - val_AUPRC: 0.5798 - val_f1_score: 0.5041 - val_balanced_accuracy: 0.6846 - val_specificity: 0.9803 - val_miss_rate: 0.6111 - val_fall_out: 0.0197 - val_mcc: 0.4870
198/198 [==============================] - 7s 34ms/step - loss: 0.8718 - accuracy: 0.9424 - recall: 0.5697 - precision: 0.8800 - AUROC: 0.9640 - AUPRC: 0.8408 - f1_score: 0.6916 - balanced_accuracy: 0.7799 - specificity: 0.9901 - miss_rate: 0.4303 - fall_out: 0.0099 - mcc: 0.6805
50/50 [==============================] - 5s 94ms/step - loss: 0.9837 - accuracy: 0.9132 - recall: 0.3889 - precision: 0.7163 - AUROC: 0.8391 - AUPRC: 0.5798 - f1_score: 0.5041 - balanced_accuracy: 0.6846 - specificity: 0.9803 - miss_rate: 0.6111 - fall_out: 0.0197 - mcc: 0.4870
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_3\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_3\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 72, 'dropout_0': 0.5}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_3\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_3\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 160, 'kernel_size_0': 16, 'conv_dropout_0': 0.2, 'dense_depth': 1, 'units_0': 128, 'dense_dropout_0': 0.3, 'filters_1': 104, 'kernel_size_1': 16, 'conv_dropout_1': 0.0, 'filters_2': 160, 'kernel_size_2': 6, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 6, 'conv_dropout_3': 0.0, 'units_1': 80, 'dense_dropout_1': 0.3, 'units_2': 128, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.3, 'units_4': 80, 'dense_dropout_4': 0.3, 'filters_4': 8, 'kernel_size_4': 6, 'conv_dropout_4': 0.0, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['SNRNP70', 'ATM']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 72) 40464
dropout_3 (Dropout) (None, 72) 0
dense_5 (Dense) (None, 1) 73
=================================================================
Total params: 40,537
Trainable params: 40,537
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 10400
dropout_4 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 104) 266344
dropout_5 (Dropout) (None, 128, 104) 0
max_pooling1d_2 (MaxPooling (None, 64, 104) 0
1D)
conv1d_3 (Conv1D) (None, 64, 160) 100000
dropout_6 (Dropout) (None, 64, 160) 0
max_pooling1d_3 (MaxPooling (None, 32, 160) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 7688
dropout_7 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 8) 392
dropout_8 (Dropout) (None, 16, 8) 0
max_pooling1d_5 (MaxPooling (None, 8, 8) 0
1D)
conv1d_6 (Conv1D) (None, 8, 8) 392
dropout_9 (Dropout) (None, 8, 8) 0
max_pooling1d_6 (MaxPooling (None, 4, 8) 0
1D)
flatten_1 (Flatten) (None, 32) 0
dense_6 (Dense) (None, 128) 4224
dropout_10 (Dropout) (None, 128) 0
dense_7 (Dense) (None, 1) 129
=================================================================
Total params: 389,569
Trainable params: 389,569
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 10400 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 104) 266344 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 104) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 104) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 160) 100000 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 160) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 160) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 7688 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_7[0][0]']
conv1d_5 (Conv1D) (None, 16, 8) 392 ['max_pooling1d_4[0][0]']
dropout_8 (Dropout) (None, 16, 8) 0 ['conv1d_5[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 8) 0 ['dropout_8[0][0]']
conv1d_6 (Conv1D) (None, 8, 8) 392 ['max_pooling1d_5[0][0]']
dropout_9 (Dropout) (None, 8, 8) 0 ['conv1d_6[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 8) 0 ['dropout_9[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
flatten_1 (Flatten) (None, 32) 0 ['max_pooling1d_6[0][0]']
dense_4 (Dense) (None, 72) 40464 ['epigenomic_data[0][0]']
dense_6 (Dense) (None, 128) 4224 ['flatten_1[0][0]']
dropout_3 (Dropout) (None, 72) 0 ['dense_4[0][0]']
dropout_10 (Dropout) (None, 128) 0 ['dense_6[0][0]']
concatenate (Concatenate) (None, 200) 0 ['dropout_3[0][0]',
'dropout_10[0][0]']
dense_8 (Dense) (None, 64) 12864 ['concatenate[0][0]']
dense_9 (Dense) (None, 1) 65 ['dense_8[0][0]']
==================================================================================================
Total params: 442,833
Trainable params: 442,833
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 14s 60ms/step - loss: 0.3001 - accuracy: 0.8957 - recall: 0.3086 - precision: 0.5748 - AUROC: 0.7804 - AUPRC: 0.4458 - f1_score: 0.4016 - balanced_accuracy: 0.6397 - specificity: 0.9708 - miss_rate: 0.6914 - fall_out: 0.0292 - mcc: 0.3705 - val_loss: 0.2439 - val_accuracy: 0.9140 - val_recall: 0.3610 - val_precision: 0.7518 - val_AUROC: 0.8544 - val_AUPRC: 0.5797 - val_f1_score: 0.4878 - val_balanced_accuracy: 0.6729 - val_specificity: 0.9848 - val_miss_rate: 0.6390 - val_fall_out: 0.0152 - val_mcc: 0.4832
Epoch 2/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2598 - accuracy: 0.9101 - recall: 0.3349 - precision: 0.7246 - AUROC: 0.8304 - AUPRC: 0.5426 - f1_score: 0.4581 - balanced_accuracy: 0.6593 - specificity: 0.9837 - miss_rate: 0.6651 - fall_out: 0.0163 - mcc: 0.4533 - val_loss: 0.2475 - val_accuracy: 0.9095 - val_recall: 0.2599 - val_precision: 0.8162 - val_AUROC: 0.8592 - val_AUPRC: 0.5729 - val_f1_score: 0.3943 - val_balanced_accuracy: 0.6262 - val_specificity: 0.9925 - val_miss_rate: 0.7401 - val_fall_out: 0.0075 - val_mcc: 0.4290
Epoch 3/1000
198/198 [==============================] - 12s 55ms/step - loss: 0.2511 - accuracy: 0.9114 - recall: 0.3386 - precision: 0.7392 - AUROC: 0.8437 - AUPRC: 0.5643 - f1_score: 0.4644 - balanced_accuracy: 0.6616 - specificity: 0.9847 - miss_rate: 0.6614 - fall_out: 0.0153 - mcc: 0.4619 - val_loss: 0.2397 - val_accuracy: 0.9129 - val_recall: 0.2899 - val_precision: 0.8320 - val_AUROC: 0.8658 - val_AUPRC: 0.5982 - val_f1_score: 0.4300 - val_balanced_accuracy: 0.6412 - val_specificity: 0.9925 - val_miss_rate: 0.7101 - val_fall_out: 0.0075 - val_mcc: 0.4597
Epoch 4/1000
198/198 [==============================] - 11s 55ms/step - loss: 0.2449 - accuracy: 0.9125 - recall: 0.3490 - precision: 0.7430 - AUROC: 0.8537 - AUPRC: 0.5816 - f1_score: 0.4749 - balanced_accuracy: 0.6668 - specificity: 0.9846 - miss_rate: 0.6510 - fall_out: 0.0154 - mcc: 0.4710 - val_loss: 0.2390 - val_accuracy: 0.9134 - val_recall: 0.2899 - val_precision: 0.8438 - val_AUROC: 0.8646 - val_AUPRC: 0.6029 - val_f1_score: 0.4315 - val_balanced_accuracy: 0.6415 - val_specificity: 0.9931 - val_miss_rate: 0.7101 - val_fall_out: 0.0069 - val_mcc: 0.4638
Epoch 5/1000
198/198 [==============================] - 11s 55ms/step - loss: 0.2417 - accuracy: 0.9144 - recall: 0.3605 - precision: 0.7580 - AUROC: 0.8584 - AUPRC: 0.5901 - f1_score: 0.4886 - balanced_accuracy: 0.6729 - specificity: 0.9853 - miss_rate: 0.6395 - fall_out: 0.0147 - mcc: 0.4854 - val_loss: 0.2374 - val_accuracy: 0.9154 - val_recall: 0.3275 - val_precision: 0.8160 - val_AUROC: 0.8666 - val_AUPRC: 0.6045 - val_f1_score: 0.4674 - val_balanced_accuracy: 0.6590 - val_specificity: 0.9906 - val_miss_rate: 0.6725 - val_fall_out: 0.0094 - val_mcc: 0.4839
Epoch 6/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2376 - accuracy: 0.9144 - recall: 0.3614 - precision: 0.7570 - AUROC: 0.8662 - AUPRC: 0.5997 - f1_score: 0.4892 - balanced_accuracy: 0.6733 - specificity: 0.9852 - miss_rate: 0.6386 - fall_out: 0.0148 - mcc: 0.4856 - val_loss: 0.2436 - val_accuracy: 0.9091 - val_recall: 0.2328 - val_precision: 0.8698 - val_AUROC: 0.8664 - val_AUPRC: 0.6052 - val_f1_score: 0.3672 - val_balanced_accuracy: 0.6141 - val_specificity: 0.9955 - val_miss_rate: 0.7672 - val_fall_out: 0.0045 - val_mcc: 0.4220
Epoch 7/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2331 - accuracy: 0.9168 - recall: 0.3755 - precision: 0.7753 - AUROC: 0.8713 - AUPRC: 0.6124 - f1_score: 0.5059 - balanced_accuracy: 0.6808 - specificity: 0.9861 - miss_rate: 0.6245 - fall_out: 0.0139 - mcc: 0.5032 - val_loss: 0.2432 - val_accuracy: 0.9106 - val_recall: 0.2432 - val_precision: 0.8858 - val_AUROC: 0.8637 - val_AUPRC: 0.6001 - val_f1_score: 0.3816 - val_balanced_accuracy: 0.6196 - val_specificity: 0.9960 - val_miss_rate: 0.7568 - val_fall_out: 0.0040 - val_mcc: 0.4367
198/198 [==============================] - 7s 33ms/step - loss: 0.2273 - accuracy: 0.9118 - recall: 0.2443 - precision: 0.9170 - AUROC: 0.8928 - AUPRC: 0.6496 - f1_score: 0.3859 - balanced_accuracy: 0.6208 - specificity: 0.9972 - miss_rate: 0.7557 - fall_out: 0.0028 - mcc: 0.4473
50/50 [==============================] - 5s 92ms/step - loss: 0.2432 - accuracy: 0.9106 - recall: 0.2432 - precision: 0.8858 - AUROC: 0.8637 - AUPRC: 0.6001 - f1_score: 0.3816 - balanced_accuracy: 0.6196 - specificity: 0.9960 - miss_rate: 0.7568 - fall_out: 0.0040 - mcc: 0.4367
-- HOLDOUT 5 --
-- 1 Uncorrelated features: [Pearson+Spearman]
['SNRNP70']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 16s 66ms/step - loss: 17.7494 - accuracy: 0.8999 - recall: 0.2501 - precision: 0.6530 - AUROC: 0.7447 - AUPRC: 0.4032 - f1_score: 0.3617 - balanced_accuracy: 0.6165 - specificity: 0.9830 - miss_rate: 0.7499 - fall_out: 0.0170 - mcc: 0.3626 - val_loss: 2.3676 - val_accuracy: 0.9103 - val_recall: 0.2516 - val_precision: 0.8555 - val_AUROC: 0.8237 - val_AUPRC: 0.5587 - val_f1_score: 0.3888 - val_balanced_accuracy: 0.6231 - val_specificity: 0.9946 - val_miss_rate: 0.7484 - val_fall_out: 0.0054 - val_mcc: 0.4347
Epoch 2/1000
198/198 [==============================] - 14s 64ms/step - loss: 2.0809 - accuracy: 0.9068 - recall: 0.3029 - precision: 0.7086 - AUROC: 0.8274 - AUPRC: 0.5213 - f1_score: 0.4244 - balanced_accuracy: 0.6435 - specificity: 0.9841 - miss_rate: 0.6971 - fall_out: 0.0159 - mcc: 0.4237 - val_loss: 1.7495 - val_accuracy: 0.9028 - val_recall: 0.1554 - val_precision: 0.9253 - val_AUROC: 0.8409 - val_AUPRC: 0.5769 - val_f1_score: 0.2661 - val_balanced_accuracy: 0.5769 - val_specificity: 0.9984 - val_miss_rate: 0.8446 - val_fall_out: 0.0016 - val_mcc: 0.3568
Epoch 3/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.8222 - accuracy: 0.9088 - recall: 0.3164 - precision: 0.7242 - AUROC: 0.8390 - AUPRC: 0.5487 - f1_score: 0.4404 - balanced_accuracy: 0.6505 - specificity: 0.9846 - miss_rate: 0.6836 - fall_out: 0.0154 - mcc: 0.4398 - val_loss: 1.5995 - val_accuracy: 0.9137 - val_recall: 0.2983 - val_precision: 0.8343 - val_AUROC: 0.8432 - val_AUPRC: 0.5819 - val_f1_score: 0.4394 - val_balanced_accuracy: 0.6453 - val_specificity: 0.9924 - val_miss_rate: 0.7017 - val_fall_out: 0.0076 - val_mcc: 0.4673
Epoch 4/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.7019 - accuracy: 0.9105 - recall: 0.3279 - precision: 0.7370 - AUROC: 0.8461 - AUPRC: 0.5618 - f1_score: 0.4539 - balanced_accuracy: 0.6565 - specificity: 0.9850 - miss_rate: 0.6721 - fall_out: 0.0150 - mcc: 0.4534 - val_loss: 1.4732 - val_accuracy: 0.9011 - val_recall: 0.1366 - val_precision: 0.9378 - val_AUROC: 0.8465 - val_AUPRC: 0.5870 - val_f1_score: 0.2384 - val_balanced_accuracy: 0.5677 - val_specificity: 0.9988 - val_miss_rate: 0.8634 - val_fall_out: 0.0012 - val_mcc: 0.3369
Epoch 5/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.6069 - accuracy: 0.9118 - recall: 0.3377 - precision: 0.7449 - AUROC: 0.8551 - AUPRC: 0.5762 - f1_score: 0.4647 - balanced_accuracy: 0.6614 - specificity: 0.9852 - miss_rate: 0.6623 - fall_out: 0.0148 - mcc: 0.4636 - val_loss: 1.3969 - val_accuracy: 0.9101 - val_recall: 0.2334 - val_precision: 0.8981 - val_AUROC: 0.8472 - val_AUPRC: 0.5904 - val_f1_score: 0.3706 - val_balanced_accuracy: 0.6150 - val_specificity: 0.9966 - val_miss_rate: 0.7666 - val_fall_out: 0.0034 - val_mcc: 0.4313
Epoch 6/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.5312 - accuracy: 0.9122 - recall: 0.3403 - precision: 0.7487 - AUROC: 0.8607 - AUPRC: 0.5832 - f1_score: 0.4679 - balanced_accuracy: 0.6628 - specificity: 0.9854 - miss_rate: 0.6597 - fall_out: 0.0146 - mcc: 0.4670 - val_loss: 1.3163 - val_accuracy: 0.9130 - val_recall: 0.2718 - val_precision: 0.8744 - val_AUROC: 0.8473 - val_AUPRC: 0.5908 - val_f1_score: 0.4147 - val_balanced_accuracy: 0.6334 - val_specificity: 0.9950 - val_miss_rate: 0.7282 - val_fall_out: 0.0050 - val_mcc: 0.4588
Epoch 7/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.4672 - accuracy: 0.9130 - recall: 0.3476 - precision: 0.7521 - AUROC: 0.8668 - AUPRC: 0.5911 - f1_score: 0.4755 - balanced_accuracy: 0.6665 - specificity: 0.9853 - miss_rate: 0.6524 - fall_out: 0.0147 - mcc: 0.4737 - val_loss: 1.2498 - val_accuracy: 0.9128 - val_recall: 0.2662 - val_precision: 0.8822 - val_AUROC: 0.8481 - val_AUPRC: 0.5934 - val_f1_score: 0.4090 - val_balanced_accuracy: 0.6308 - val_specificity: 0.9955 - val_miss_rate: 0.7338 - val_fall_out: 0.0045 - val_mcc: 0.4564
Epoch 8/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.4168 - accuracy: 0.9143 - recall: 0.3567 - precision: 0.7611 - AUROC: 0.8658 - AUPRC: 0.5942 - f1_score: 0.4857 - balanced_accuracy: 0.6712 - specificity: 0.9857 - miss_rate: 0.6433 - fall_out: 0.0143 - mcc: 0.4839 - val_loss: 1.2134 - val_accuracy: 0.9076 - val_recall: 0.2000 - val_precision: 0.9288 - val_AUROC: 0.8470 - val_AUPRC: 0.5912 - val_f1_score: 0.3291 - val_balanced_accuracy: 0.5990 - val_specificity: 0.9980 - val_miss_rate: 0.8000 - val_fall_out: 0.0020 - val_mcc: 0.4069
Epoch 9/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.3696 - accuracy: 0.9152 - recall: 0.3692 - precision: 0.7596 - AUROC: 0.8720 - AUPRC: 0.6057 - f1_score: 0.4969 - balanced_accuracy: 0.6771 - specificity: 0.9851 - miss_rate: 0.6308 - fall_out: 0.0149 - mcc: 0.4922 - val_loss: 1.2111 - val_accuracy: 0.9085 - val_recall: 0.2111 - val_precision: 0.9210 - val_AUROC: 0.8482 - val_AUPRC: 0.5906 - val_f1_score: 0.3435 - val_balanced_accuracy: 0.6044 - val_specificity: 0.9977 - val_miss_rate: 0.7889 - val_fall_out: 0.0023 - val_mcc: 0.4161
Epoch 10/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.3322 - accuracy: 0.9150 - recall: 0.3656 - precision: 0.7605 - AUROC: 0.8743 - AUPRC: 0.6093 - f1_score: 0.4938 - balanced_accuracy: 0.6754 - specificity: 0.9853 - miss_rate: 0.6344 - fall_out: 0.0147 - mcc: 0.4900 - val_loss: 1.1432 - val_accuracy: 0.9129 - val_recall: 0.2794 - val_precision: 0.8532 - val_AUROC: 0.8511 - val_AUPRC: 0.5950 - val_f1_score: 0.4210 - val_balanced_accuracy: 0.6366 - val_specificity: 0.9939 - val_miss_rate: 0.7206 - val_fall_out: 0.0061 - val_mcc: 0.4582
Epoch 11/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.2991 - accuracy: 0.9143 - recall: 0.3610 - precision: 0.7563 - AUROC: 0.8761 - AUPRC: 0.6098 - f1_score: 0.4887 - balanced_accuracy: 0.6731 - specificity: 0.9851 - miss_rate: 0.6390 - fall_out: 0.0149 - mcc: 0.4850 - val_loss: 1.1489 - val_accuracy: 0.9071 - val_recall: 0.1951 - val_precision: 0.9302 - val_AUROC: 0.8493 - val_AUPRC: 0.5917 - val_f1_score: 0.3226 - val_balanced_accuracy: 0.5966 - val_specificity: 0.9981 - val_miss_rate: 0.8049 - val_fall_out: 0.0019 - val_mcc: 0.4021
Epoch 12/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.2668 - accuracy: 0.9156 - recall: 0.3776 - precision: 0.7562 - AUROC: 0.8821 - AUPRC: 0.6218 - f1_score: 0.5037 - balanced_accuracy: 0.6810 - specificity: 0.9844 - miss_rate: 0.6224 - fall_out: 0.0156 - mcc: 0.4966 - val_loss: 1.1524 - val_accuracy: 0.9138 - val_recall: 0.2850 - val_precision: 0.8629 - val_AUROC: 0.8498 - val_AUPRC: 0.5952 - val_f1_score: 0.4285 - val_balanced_accuracy: 0.6396 - val_specificity: 0.9942 - val_miss_rate: 0.7150 - val_fall_out: 0.0058 - val_mcc: 0.4663
198/198 [==============================] - 7s 36ms/step - loss: 1.1261 - accuracy: 0.9151 - recall: 0.2952 - precision: 0.8715 - AUROC: 0.9013 - AUPRC: 0.6640 - f1_score: 0.4410 - balanced_accuracy: 0.6448 - specificity: 0.9944 - miss_rate: 0.7048 - fall_out: 0.0056 - mcc: 0.4778
50/50 [==============================] - 5s 95ms/step - loss: 1.1524 - accuracy: 0.9138 - recall: 0.2850 - precision: 0.8629 - AUROC: 0.8498 - AUPRC: 0.5952 - f1_score: 0.4285 - balanced_accuracy: 0.6396 - specificity: 0.9942 - miss_rate: 0.7150 - fall_out: 0.0058 - mcc: 0.4663
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_4\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_4\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 1, 'units_0': 72, 'dropout_0': 0.5, 'units_1': 8, 'dropout_1': 0.5, 'units_2': 8, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.4, 'units_4': 104, 'dropout_4': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_4\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_4\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 104, 'kernel_size_0': 16, 'conv_dropout_0': 0.2, 'dense_depth': 4, 'units_0': 96, 'dense_dropout_0': 0.3, 'filters_1': 128, 'kernel_size_1': 8, 'conv_dropout_1': 0.2, 'filters_2': 32, 'kernel_size_2': 8, 'conv_dropout_2': 0.0, 'filters_3': 16, 'kernel_size_3': 8, 'conv_dropout_3': 0.0, 'filters_4': 16, 'kernel_size_4': 8, 'conv_dropout_4': 0.0, 'filters_5': 16, 'kernel_size_5': 6, 'conv_dropout_5': 0.0, 'units_1': 32, 'dense_dropout_1': 0.3, 'units_2': 80, 'dense_dropout_2': 0.5, 'units_3': 32, 'dense_dropout_3': 0.3}
- Removing uncorrelated/correlated features: ['SNRNP70']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 72) 40464
dropout_3 (Dropout) (None, 72) 0
dense_5 (Dense) (None, 1) 73
=================================================================
Total params: 40,537
Trainable params: 40,537
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 104) 6760
dropout_4 (Dropout) (None, 256, 104) 0
max_pooling1d_1 (MaxPooling (None, 128, 104) 0
1D)
conv1d_2 (Conv1D) (None, 128, 128) 106624
dropout_5 (Dropout) (None, 128, 128) 0
max_pooling1d_2 (MaxPooling (None, 64, 128) 0
1D)
conv1d_3 (Conv1D) (None, 64, 32) 32800
dropout_6 (Dropout) (None, 64, 32) 0
max_pooling1d_3 (MaxPooling (None, 32, 32) 0
1D)
conv1d_4 (Conv1D) (None, 32, 16) 4112
dropout_7 (Dropout) (None, 32, 16) 0
max_pooling1d_4 (MaxPooling (None, 16, 16) 0
1D)
conv1d_5 (Conv1D) (None, 16, 16) 2064
dropout_8 (Dropout) (None, 16, 16) 0
max_pooling1d_5 (MaxPooling (None, 8, 16) 0
1D)
conv1d_6 (Conv1D) (None, 8, 16) 1552
dropout_9 (Dropout) (None, 8, 16) 0
max_pooling1d_6 (MaxPooling (None, 4, 16) 0
1D)
flatten_1 (Flatten) (None, 64) 0
dense_6 (Dense) (None, 96) 6240
dropout_10 (Dropout) (None, 96) 0
dense_7 (Dense) (None, 32) 3104
dropout_11 (Dropout) (None, 32) 0
dense_8 (Dense) (None, 80) 2640
dropout_12 (Dropout) (None, 80) 0
dense_9 (Dense) (None, 32) 2592
dropout_13 (Dropout) (None, 32) 0
dense_10 (Dense) (None, 1) 33
=================================================================
Total params: 168,521
Trainable params: 168,521
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 104) 6760 ['sequence_data[0][0]']
dropout_4 (Dropout) (None, 256, 104) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 104) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 128, 128) 106624 ['max_pooling1d_1[0][0]']
dropout_5 (Dropout) (None, 128, 128) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 128) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 64, 32) 32800 ['max_pooling1d_2[0][0]']
dropout_6 (Dropout) (None, 64, 32) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 32) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 32, 16) 4112 ['max_pooling1d_3[0][0]']
dropout_7 (Dropout) (None, 32, 16) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 16) 0 ['dropout_7[0][0]']
conv1d_5 (Conv1D) (None, 16, 16) 2064 ['max_pooling1d_4[0][0]']
dropout_8 (Dropout) (None, 16, 16) 0 ['conv1d_5[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 16) 0 ['dropout_8[0][0]']
conv1d_6 (Conv1D) (None, 8, 16) 1552 ['max_pooling1d_5[0][0]']
dropout_9 (Dropout) (None, 8, 16) 0 ['conv1d_6[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 16) 0 ['dropout_9[0][0]']
flatten_1 (Flatten) (None, 64) 0 ['max_pooling1d_6[0][0]']
dense_6 (Dense) (None, 96) 6240 ['flatten_1[0][0]']
dropout_10 (Dropout) (None, 96) 0 ['dense_6[0][0]']
dense_7 (Dense) (None, 32) 3104 ['dropout_10[0][0]']
dropout_11 (Dropout) (None, 32) 0 ['dense_7[0][0]']
dense_8 (Dense) (None, 80) 2640 ['dropout_11[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
dropout_12 (Dropout) (None, 80) 0 ['dense_8[0][0]']
dense_4 (Dense) (None, 72) 40464 ['epigenomic_data[0][0]']
dense_9 (Dense) (None, 32) 2592 ['dropout_12[0][0]']
dropout_3 (Dropout) (None, 72) 0 ['dense_4[0][0]']
dropout_13 (Dropout) (None, 32) 0 ['dense_9[0][0]']
concatenate (Concatenate) (None, 104) 0 ['dropout_3[0][0]',
'dropout_13[0][0]']
dense_11 (Dense) (None, 64) 6720 ['concatenate[0][0]']
dense_12 (Dense) (None, 1) 65 ['dense_11[0][0]']
==================================================================================================
Total params: 215,737
Trainable params: 215,737
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 16s 66ms/step - loss: 0.3399 - accuracy: 0.8756 - recall: 0.3142 - precision: 0.4332 - AUROC: 0.7562 - AUPRC: 0.3863 - f1_score: 0.3642 - balanced_accuracy: 0.6308 - specificity: 0.9474 - miss_rate: 0.6858 - fall_out: 0.0526 - mcc: 0.3019 - val_loss: 0.2707 - val_accuracy: 0.9111 - val_recall: 0.3610 - val_precision: 0.7135 - val_AUROC: 0.8226 - val_AUPRC: 0.5532 - val_f1_score: 0.4794 - val_balanced_accuracy: 0.6712 - val_specificity: 0.9815 - val_miss_rate: 0.6390 - val_fall_out: 0.0185 - val_mcc: 0.4669
Epoch 2/1000
198/198 [==============================] - 12s 56ms/step - loss: 0.2634 - accuracy: 0.9089 - recall: 0.3318 - precision: 0.7103 - AUROC: 0.8313 - AUPRC: 0.5361 - f1_score: 0.4523 - balanced_accuracy: 0.6572 - specificity: 0.9827 - miss_rate: 0.6682 - fall_out: 0.0173 - mcc: 0.4452 - val_loss: 0.2557 - val_accuracy: 0.9129 - val_recall: 0.3470 - val_precision: 0.7500 - val_AUROC: 0.8363 - val_AUPRC: 0.5697 - val_f1_score: 0.4745 - val_balanced_accuracy: 0.6661 - val_specificity: 0.9852 - val_miss_rate: 0.6530 - val_fall_out: 0.0148 - val_mcc: 0.4725
Epoch 3/1000
198/198 [==============================] - 12s 57ms/step - loss: 0.2521 - accuracy: 0.9113 - recall: 0.3372 - precision: 0.7386 - AUROC: 0.8467 - AUPRC: 0.5644 - f1_score: 0.4630 - balanced_accuracy: 0.6610 - specificity: 0.9847 - miss_rate: 0.6628 - fall_out: 0.0153 - mcc: 0.4607 - val_loss: 0.2615 - val_accuracy: 0.9086 - val_recall: 0.2230 - val_precision: 0.8840 - val_AUROC: 0.8363 - val_AUPRC: 0.5678 - val_f1_score: 0.3561 - val_balanced_accuracy: 0.6096 - val_specificity: 0.9963 - val_miss_rate: 0.7770 - val_fall_out: 0.0037 - val_mcc: 0.4171
Epoch 4/1000
198/198 [==============================] - 12s 57ms/step - loss: 0.2463 - accuracy: 0.9127 - recall: 0.3462 - precision: 0.7491 - AUROC: 0.8551 - AUPRC: 0.5758 - f1_score: 0.4736 - balanced_accuracy: 0.6657 - specificity: 0.9852 - miss_rate: 0.6538 - fall_out: 0.0148 - mcc: 0.4715 - val_loss: 0.2500 - val_accuracy: 0.9132 - val_recall: 0.3359 - val_precision: 0.7675 - val_AUROC: 0.8440 - val_AUPRC: 0.5830 - val_f1_score: 0.4673 - val_balanced_accuracy: 0.6614 - val_specificity: 0.9870 - val_miss_rate: 0.6641 - val_fall_out: 0.0130 - val_mcc: 0.4714
Epoch 5/1000
198/198 [==============================] - 12s 57ms/step - loss: 0.2429 - accuracy: 0.9129 - recall: 0.3466 - precision: 0.7512 - AUROC: 0.8601 - AUPRC: 0.5846 - f1_score: 0.4743 - balanced_accuracy: 0.6659 - specificity: 0.9853 - miss_rate: 0.6534 - fall_out: 0.0147 - mcc: 0.4726 - val_loss: 0.2535 - val_accuracy: 0.9132 - val_recall: 0.2892 - val_precision: 0.8401 - val_AUROC: 0.8436 - val_AUPRC: 0.5837 - val_f1_score: 0.4303 - val_balanced_accuracy: 0.6411 - val_specificity: 0.9930 - val_miss_rate: 0.7108 - val_fall_out: 0.0070 - val_mcc: 0.4619
Epoch 6/1000
198/198 [==============================] - 12s 58ms/step - loss: 0.2384 - accuracy: 0.9141 - recall: 0.3621 - precision: 0.7519 - AUROC: 0.8671 - AUPRC: 0.5957 - f1_score: 0.4888 - balanced_accuracy: 0.6734 - specificity: 0.9847 - miss_rate: 0.6379 - fall_out: 0.0153 - mcc: 0.4840 - val_loss: 0.2504 - val_accuracy: 0.9144 - val_recall: 0.3073 - val_precision: 0.8305 - val_AUROC: 0.8449 - val_AUPRC: 0.5856 - val_f1_score: 0.4486 - val_balanced_accuracy: 0.6496 - val_specificity: 0.9920 - val_miss_rate: 0.6927 - val_fall_out: 0.0080 - val_mcc: 0.4733
198/198 [==============================] - 7s 37ms/step - loss: 0.2247 - accuracy: 0.9153 - recall: 0.3166 - precision: 0.8332 - AUROC: 0.8889 - AUPRC: 0.6349 - f1_score: 0.4589 - balanced_accuracy: 0.6543 - specificity: 0.9919 - miss_rate: 0.6834 - fall_out: 0.0081 - mcc: 0.4817
50/50 [==============================] - 5s 96ms/step - loss: 0.2504 - accuracy: 0.9144 - recall: 0.3073 - precision: 0.8305 - AUROC: 0.8449 - AUPRC: 0.5856 - f1_score: 0.4486 - balanced_accuracy: 0.6496 - specificity: 0.9920 - miss_rate: 0.6927 - fall_out: 0.0080 - mcc: 0.4733
-- HOLDOUT 6 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['SNRNP70', 'ZNF282']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70', 'ZNF282']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 16s 67ms/step - loss: 16.4572 - accuracy: 0.8827 - recall: 0.2752 - precision: 0.4707 - AUROC: 0.7426 - AUPRC: 0.3833 - f1_score: 0.3473 - balanced_accuracy: 0.6178 - specificity: 0.9604 - miss_rate: 0.7248 - fall_out: 0.0396 - mcc: 0.3002 - val_loss: 2.4133 - val_accuracy: 0.8994 - val_recall: 0.1345 - val_precision: 0.8616 - val_AUROC: 0.8294 - val_AUPRC: 0.5528 - val_f1_score: 0.2327 - val_balanced_accuracy: 0.5659 - val_specificity: 0.9972 - val_miss_rate: 0.8655 - val_fall_out: 0.0028 - val_mcc: 0.3168
Epoch 2/1000
198/198 [==============================] - 13s 65ms/step - loss: 2.0263 - accuracy: 0.9085 - recall: 0.3133 - precision: 0.7237 - AUROC: 0.8233 - AUPRC: 0.5304 - f1_score: 0.4373 - balanced_accuracy: 0.6490 - specificity: 0.9847 - miss_rate: 0.6867 - fall_out: 0.0153 - mcc: 0.4373 - val_loss: 2.0737 - val_accuracy: 0.9023 - val_recall: 0.1561 - val_precision: 0.8996 - val_AUROC: 0.8453 - val_AUPRC: 0.5760 - val_f1_score: 0.2660 - val_balanced_accuracy: 0.5769 - val_specificity: 0.9978 - val_miss_rate: 0.8439 - val_fall_out: 0.0022 - val_mcc: 0.3513
Epoch 3/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.8177 - accuracy: 0.9097 - recall: 0.3251 - precision: 0.7276 - AUROC: 0.8410 - AUPRC: 0.5549 - f1_score: 0.4494 - balanced_accuracy: 0.6548 - specificity: 0.9844 - miss_rate: 0.6749 - fall_out: 0.0156 - mcc: 0.4475 - val_loss: 1.9250 - val_accuracy: 0.9043 - val_recall: 0.1791 - val_precision: 0.8862 - val_AUROC: 0.8541 - val_AUPRC: 0.5852 - val_f1_score: 0.2980 - val_balanced_accuracy: 0.5881 - val_specificity: 0.9971 - val_miss_rate: 0.8209 - val_fall_out: 0.0029 - val_mcc: 0.3733
Epoch 4/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.6953 - accuracy: 0.9109 - recall: 0.3314 - precision: 0.7393 - AUROC: 0.8492 - AUPRC: 0.5663 - f1_score: 0.4577 - balanced_accuracy: 0.6582 - specificity: 0.9851 - miss_rate: 0.6686 - fall_out: 0.0149 - mcc: 0.4568 - val_loss: 1.8083 - val_accuracy: 0.9033 - val_recall: 0.1638 - val_precision: 0.9073 - val_AUROC: 0.8569 - val_AUPRC: 0.5836 - val_f1_score: 0.2774 - val_balanced_accuracy: 0.5808 - val_specificity: 0.9979 - val_miss_rate: 0.8362 - val_fall_out: 0.0021 - val_mcc: 0.3619
Epoch 5/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.6008 - accuracy: 0.9121 - recall: 0.3424 - precision: 0.7444 - AUROC: 0.8545 - AUPRC: 0.5792 - f1_score: 0.4690 - balanced_accuracy: 0.6637 - specificity: 0.9850 - miss_rate: 0.6576 - fall_out: 0.0150 - mcc: 0.4668 - val_loss: 1.7325 - val_accuracy: 0.9084 - val_recall: 0.2230 - val_precision: 0.8767 - val_AUROC: 0.8560 - val_AUPRC: 0.5867 - val_f1_score: 0.3556 - val_balanced_accuracy: 0.6095 - val_specificity: 0.9960 - val_miss_rate: 0.7770 - val_fall_out: 0.0040 - val_mcc: 0.4149
Epoch 6/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.5275 - accuracy: 0.9122 - recall: 0.3391 - precision: 0.7491 - AUROC: 0.8576 - AUPRC: 0.5811 - f1_score: 0.4669 - balanced_accuracy: 0.6623 - specificity: 0.9855 - miss_rate: 0.6609 - fall_out: 0.0145 - mcc: 0.4664 - val_loss: 1.6411 - val_accuracy: 0.9122 - val_recall: 0.2767 - val_precision: 0.8447 - val_AUROC: 0.8568 - val_AUPRC: 0.5906 - val_f1_score: 0.4168 - val_balanced_accuracy: 0.6351 - val_specificity: 0.9935 - val_miss_rate: 0.7233 - val_fall_out: 0.0065 - val_mcc: 0.4530
Epoch 7/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.4677 - accuracy: 0.9123 - recall: 0.3394 - precision: 0.7502 - AUROC: 0.8627 - AUPRC: 0.5886 - f1_score: 0.4674 - balanced_accuracy: 0.6625 - specificity: 0.9855 - miss_rate: 0.6606 - fall_out: 0.0145 - mcc: 0.4670 - val_loss: 1.6130 - val_accuracy: 0.8984 - val_recall: 0.1122 - val_precision: 0.9306 - val_AUROC: 0.8551 - val_AUPRC: 0.5898 - val_f1_score: 0.2002 - val_balanced_accuracy: 0.5556 - val_specificity: 0.9989 - val_miss_rate: 0.8878 - val_fall_out: 0.0011 - val_mcc: 0.3034
Epoch 8/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.4186 - accuracy: 0.9134 - recall: 0.3448 - precision: 0.7610 - AUROC: 0.8636 - AUPRC: 0.5953 - f1_score: 0.4746 - balanced_accuracy: 0.6655 - specificity: 0.9861 - miss_rate: 0.6552 - fall_out: 0.0139 - mcc: 0.4753 - val_loss: 1.5464 - val_accuracy: 0.9096 - val_recall: 0.2307 - val_precision: 0.8922 - val_AUROC: 0.8563 - val_AUPRC: 0.5951 - val_f1_score: 0.3666 - val_balanced_accuracy: 0.6135 - val_specificity: 0.9964 - val_miss_rate: 0.7693 - val_fall_out: 0.0036 - val_mcc: 0.4269
Epoch 9/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.3750 - accuracy: 0.9145 - recall: 0.3608 - precision: 0.7593 - AUROC: 0.8704 - AUPRC: 0.6029 - f1_score: 0.4892 - balanced_accuracy: 0.6731 - specificity: 0.9854 - miss_rate: 0.6392 - fall_out: 0.0146 - mcc: 0.4861 - val_loss: 1.5956 - val_accuracy: 0.9073 - val_recall: 0.2070 - val_precision: 0.8946 - val_AUROC: 0.8535 - val_AUPRC: 0.5919 - val_f1_score: 0.3362 - val_balanced_accuracy: 0.6019 - val_specificity: 0.9969 - val_miss_rate: 0.7930 - val_fall_out: 0.0031 - val_mcc: 0.4044
Epoch 10/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.3366 - accuracy: 0.9149 - recall: 0.3647 - precision: 0.7601 - AUROC: 0.8730 - AUPRC: 0.6091 - f1_score: 0.4929 - balanced_accuracy: 0.6750 - specificity: 0.9853 - miss_rate: 0.6353 - fall_out: 0.0147 - mcc: 0.4892 - val_loss: 1.5371 - val_accuracy: 0.9073 - val_recall: 0.2007 - val_precision: 0.9172 - val_AUROC: 0.8534 - val_AUPRC: 0.5945 - val_f1_score: 0.3293 - val_balanced_accuracy: 0.5992 - val_specificity: 0.9977 - val_miss_rate: 0.7993 - val_fall_out: 0.0023 - val_mcc: 0.4044
Epoch 11/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.3031 - accuracy: 0.9140 - recall: 0.3570 - precision: 0.7559 - AUROC: 0.8767 - AUPRC: 0.6112 - f1_score: 0.4850 - balanced_accuracy: 0.6711 - specificity: 0.9853 - miss_rate: 0.6430 - fall_out: 0.0147 - mcc: 0.4820 - val_loss: 1.5047 - val_accuracy: 0.9114 - val_recall: 0.2585 - val_precision: 0.8648 - val_AUROC: 0.8557 - val_AUPRC: 0.5981 - val_f1_score: 0.3981 - val_balanced_accuracy: 0.6267 - val_specificity: 0.9948 - val_miss_rate: 0.7415 - val_fall_out: 0.0052 - val_mcc: 0.4439
Epoch 12/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.2703 - accuracy: 0.9158 - recall: 0.3767 - precision: 0.7592 - AUROC: 0.8815 - AUPRC: 0.6211 - f1_score: 0.5036 - balanced_accuracy: 0.6807 - specificity: 0.9847 - miss_rate: 0.6233 - fall_out: 0.0153 - mcc: 0.4973 - val_loss: 1.3937 - val_accuracy: 0.9121 - val_recall: 0.2690 - val_precision: 0.8597 - val_AUROC: 0.8516 - val_AUPRC: 0.5918 - val_f1_score: 0.4098 - val_balanced_accuracy: 0.6317 - val_specificity: 0.9944 - val_miss_rate: 0.7310 - val_fall_out: 0.0056 - val_mcc: 0.4514
Epoch 13/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.2431 - accuracy: 0.9163 - recall: 0.3739 - precision: 0.7690 - AUROC: 0.8829 - AUPRC: 0.6229 - f1_score: 0.5032 - balanced_accuracy: 0.6798 - specificity: 0.9856 - miss_rate: 0.6261 - fall_out: 0.0144 - mcc: 0.4995 - val_loss: 1.4166 - val_accuracy: 0.9140 - val_recall: 0.3038 - val_precision: 0.8289 - val_AUROC: 0.8552 - val_AUPRC: 0.5970 - val_f1_score: 0.4447 - val_balanced_accuracy: 0.6479 - val_specificity: 0.9920 - val_miss_rate: 0.6962 - val_fall_out: 0.0080 - val_mcc: 0.4699
Epoch 14/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.2169 - accuracy: 0.9157 - recall: 0.3814 - precision: 0.7534 - AUROC: 0.8858 - AUPRC: 0.6302 - f1_score: 0.5064 - balanced_accuracy: 0.6827 - specificity: 0.9840 - miss_rate: 0.6186 - fall_out: 0.0160 - mcc: 0.4981 - val_loss: 1.3847 - val_accuracy: 0.9062 - val_recall: 0.1930 - val_precision: 0.9052 - val_AUROC: 0.8528 - val_AUPRC: 0.5892 - val_f1_score: 0.3182 - val_balanced_accuracy: 0.5952 - val_specificity: 0.9974 - val_miss_rate: 0.8070 - val_fall_out: 0.0026 - val_mcc: 0.3931
Epoch 15/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.1945 - accuracy: 0.9159 - recall: 0.3798 - precision: 0.7576 - AUROC: 0.8868 - AUPRC: 0.6289 - f1_score: 0.5060 - balanced_accuracy: 0.6821 - specificity: 0.9844 - miss_rate: 0.6202 - fall_out: 0.0156 - mcc: 0.4988 - val_loss: 1.3635 - val_accuracy: 0.9078 - val_recall: 0.2111 - val_precision: 0.8964 - val_AUROC: 0.8532 - val_AUPRC: 0.5941 - val_f1_score: 0.3418 - val_balanced_accuracy: 0.6040 - val_specificity: 0.9969 - val_miss_rate: 0.7889 - val_fall_out: 0.0031 - val_mcc: 0.4091
Epoch 16/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.1685 - accuracy: 0.9173 - recall: 0.3915 - precision: 0.7638 - AUROC: 0.8936 - AUPRC: 0.6417 - f1_score: 0.5177 - balanced_accuracy: 0.6880 - specificity: 0.9845 - miss_rate: 0.6085 - fall_out: 0.0155 - mcc: 0.5096 - val_loss: 1.2909 - val_accuracy: 0.9123 - val_recall: 0.2739 - val_precision: 0.8525 - val_AUROC: 0.8494 - val_AUPRC: 0.5932 - val_f1_score: 0.4146 - val_balanced_accuracy: 0.6339 - val_specificity: 0.9939 - val_miss_rate: 0.7261 - val_fall_out: 0.0061 - val_mcc: 0.4532
Epoch 17/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.1473 - accuracy: 0.9178 - recall: 0.3964 - precision: 0.7661 - AUROC: 0.8949 - AUPRC: 0.6450 - f1_score: 0.5224 - balanced_accuracy: 0.6904 - specificity: 0.9845 - miss_rate: 0.6036 - fall_out: 0.0155 - mcc: 0.5139 - val_loss: 1.2872 - val_accuracy: 0.9075 - val_recall: 0.2042 - val_precision: 0.9099 - val_AUROC: 0.8496 - val_AUPRC: 0.5935 - val_f1_score: 0.3335 - val_balanced_accuracy: 0.6008 - val_specificity: 0.9974 - val_miss_rate: 0.7958 - val_fall_out: 0.0026 - val_mcc: 0.4059
Epoch 18/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.1260 - accuracy: 0.9171 - recall: 0.4028 - precision: 0.7512 - AUROC: 0.8984 - AUPRC: 0.6493 - f1_score: 0.5244 - balanced_accuracy: 0.6929 - specificity: 0.9829 - miss_rate: 0.5972 - fall_out: 0.0171 - mcc: 0.5118 - val_loss: 1.2078 - val_accuracy: 0.9134 - val_recall: 0.2955 - val_precision: 0.8330 - val_AUROC: 0.8510 - val_AUPRC: 0.5966 - val_f1_score: 0.4362 - val_balanced_accuracy: 0.6439 - val_specificity: 0.9924 - val_miss_rate: 0.7045 - val_fall_out: 0.0076 - val_mcc: 0.4646
Epoch 19/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.1043 - accuracy: 0.9183 - recall: 0.4234 - precision: 0.7462 - AUROC: 0.9005 - AUPRC: 0.6572 - f1_score: 0.5402 - balanced_accuracy: 0.7025 - specificity: 0.9816 - miss_rate: 0.5766 - fall_out: 0.0184 - mcc: 0.5233 - val_loss: 1.1969 - val_accuracy: 0.9069 - val_recall: 0.1986 - val_precision: 0.9105 - val_AUROC: 0.8500 - val_AUPRC: 0.5938 - val_f1_score: 0.3261 - val_balanced_accuracy: 0.5981 - val_specificity: 0.9975 - val_miss_rate: 0.8014 - val_fall_out: 0.0025 - val_mcc: 0.4004
Epoch 20/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.0843 - accuracy: 0.9170 - recall: 0.4140 - precision: 0.7400 - AUROC: 0.9047 - AUPRC: 0.6599 - f1_score: 0.5309 - balanced_accuracy: 0.6977 - specificity: 0.9814 - miss_rate: 0.5860 - fall_out: 0.0186 - mcc: 0.5143 - val_loss: 1.1570 - val_accuracy: 0.9150 - val_recall: 0.3331 - val_precision: 0.8007 - val_AUROC: 0.8462 - val_AUPRC: 0.5932 - val_f1_score: 0.4705 - val_balanced_accuracy: 0.6612 - val_specificity: 0.9894 - val_miss_rate: 0.6669 - val_fall_out: 0.0106 - val_mcc: 0.4823
Epoch 21/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.0636 - accuracy: 0.9180 - recall: 0.4443 - precision: 0.7268 - AUROC: 0.9072 - AUPRC: 0.6676 - f1_score: 0.5514 - balanced_accuracy: 0.7115 - specificity: 0.9786 - miss_rate: 0.5557 - fall_out: 0.0214 - mcc: 0.5279 - val_loss: 1.2132 - val_accuracy: 0.9128 - val_recall: 0.2753 - val_precision: 0.8606 - val_AUROC: 0.8420 - val_AUPRC: 0.5891 - val_f1_score: 0.4171 - val_balanced_accuracy: 0.6348 - val_specificity: 0.9943 - val_miss_rate: 0.7247 - val_fall_out: 0.0057 - val_mcc: 0.4572
Epoch 22/1000
198/198 [==============================] - 13s 63ms/step - loss: 1.0464 - accuracy: 0.9189 - recall: 0.4476 - precision: 0.7330 - AUROC: 0.9096 - AUPRC: 0.6699 - f1_score: 0.5558 - balanced_accuracy: 0.7134 - specificity: 0.9791 - miss_rate: 0.5524 - fall_out: 0.0209 - mcc: 0.5330 - val_loss: 1.1404 - val_accuracy: 0.9143 - val_recall: 0.3136 - val_precision: 0.8182 - val_AUROC: 0.8424 - val_AUPRC: 0.5905 - val_f1_score: 0.4534 - val_balanced_accuracy: 0.6523 - val_specificity: 0.9911 - val_miss_rate: 0.6864 - val_fall_out: 0.0089 - val_mcc: 0.4738
Epoch 23/1000
198/198 [==============================] - 13s 64ms/step - loss: 1.0259 - accuracy: 0.9207 - recall: 0.4688 - precision: 0.7363 - AUROC: 0.9136 - AUPRC: 0.6827 - f1_score: 0.5729 - balanced_accuracy: 0.7237 - specificity: 0.9785 - miss_rate: 0.5312 - fall_out: 0.0215 - mcc: 0.5480 - val_loss: 0.9849 - val_accuracy: 0.9133 - val_recall: 0.3206 - val_precision: 0.7904 - val_AUROC: 0.8433 - val_AUPRC: 0.5858 - val_f1_score: 0.4561 - val_balanced_accuracy: 0.6548 - val_specificity: 0.9891 - val_miss_rate: 0.6794 - val_fall_out: 0.0109 - val_mcc: 0.4688
Epoch 24/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.0078 - accuracy: 0.9207 - recall: 0.4835 - precision: 0.7254 - AUROC: 0.9168 - AUPRC: 0.6849 - f1_score: 0.5802 - balanced_accuracy: 0.7300 - specificity: 0.9766 - miss_rate: 0.5165 - fall_out: 0.0234 - mcc: 0.5519 - val_loss: 1.0442 - val_accuracy: 0.9150 - val_recall: 0.3338 - val_precision: 0.7997 - val_AUROC: 0.8436 - val_AUPRC: 0.5898 - val_f1_score: 0.4710 - val_balanced_accuracy: 0.6616 - val_specificity: 0.9893 - val_miss_rate: 0.6662 - val_fall_out: 0.0107 - val_mcc: 0.4825
Epoch 25/1000
198/198 [==============================] - 13s 64ms/step - loss: 0.9901 - accuracy: 0.9226 - recall: 0.5212 - precision: 0.7186 - AUROC: 0.9190 - AUPRC: 0.6979 - f1_score: 0.6042 - balanced_accuracy: 0.7476 - specificity: 0.9739 - miss_rate: 0.4788 - fall_out: 0.0261 - mcc: 0.5714 - val_loss: 1.0599 - val_accuracy: 0.9151 - val_recall: 0.3693 - val_precision: 0.7571 - val_AUROC: 0.8407 - val_AUPRC: 0.5869 - val_f1_score: 0.4965 - val_balanced_accuracy: 0.6771 - val_specificity: 0.9849 - val_miss_rate: 0.6307 - val_fall_out: 0.0151 - val_mcc: 0.4913
198/198 [==============================] - 7s 37ms/step - loss: 0.9817 - accuracy: 0.9255 - recall: 0.4335 - precision: 0.8280 - AUROC: 0.9494 - AUPRC: 0.7691 - f1_score: 0.5690 - balanced_accuracy: 0.7110 - specificity: 0.9885 - miss_rate: 0.5665 - fall_out: 0.0115 - mcc: 0.5662
50/50 [==============================] - 5s 99ms/step - loss: 1.0599 - accuracy: 0.9151 - recall: 0.3693 - precision: 0.7571 - AUROC: 0.8407 - AUPRC: 0.5869 - f1_score: 0.4965 - balanced_accuracy: 0.6771 - specificity: 0.9849 - miss_rate: 0.6307 - fall_out: 0.0151 - mcc: 0.4913
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_5\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_5\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 3, 'units_0': 136, 'dropout_0': 0.4, 'units_1': 8, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.5, 'units_3': 136, 'dropout_3': 0.4}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_5\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_5\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 5, 'filters_0': 160, 'kernel_size_0': 6, 'conv_dropout_0': 0.0, 'dense_depth': 1, 'units_0': 32, 'dense_dropout_0': 0.4, 'filters_1': 160, 'kernel_size_1': 8, 'conv_dropout_1': 0.0, 'filters_2': 112, 'kernel_size_2': 16, 'conv_dropout_2': 0.0, 'filters_3': 8, 'kernel_size_3': 16, 'conv_dropout_3': 0.0, 'filters_4': 32, 'kernel_size_4': 16, 'conv_dropout_4': 0.0, 'units_1': 128, 'dense_dropout_1': 0.5, 'units_2': 64, 'dense_dropout_2': 0.3, 'units_3': 64, 'dense_dropout_3': 0.5, 'units_4': 16, 'dense_dropout_4': 0.4, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['SNRNP70', 'ZNF282']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 8) 1096
dropout_4 (Dropout) (None, 8) 0
dense_6 (Dense) (None, 8) 72
dropout_5 (Dropout) (None, 8) 0
dense_7 (Dense) (None, 1) 9
=================================================================
Total params: 77,609
Trainable params: 77,609
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 4000
dropout_6 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 160) 204960
dropout_7 (Dropout) (None, 128, 160) 0
max_pooling1d_2 (MaxPooling (None, 64, 160) 0
1D)
conv1d_3 (Conv1D) (None, 64, 112) 286832
dropout_8 (Dropout) (None, 64, 112) 0
max_pooling1d_3 (MaxPooling (None, 32, 112) 0
1D)
conv1d_4 (Conv1D) (None, 32, 8) 14344
dropout_9 (Dropout) (None, 32, 8) 0
max_pooling1d_4 (MaxPooling (None, 16, 8) 0
1D)
conv1d_5 (Conv1D) (None, 16, 32) 4128
dropout_10 (Dropout) (None, 16, 32) 0
max_pooling1d_5 (MaxPooling (None, 8, 32) 0
1D)
flatten_1 (Flatten) (None, 256) 0
dense_8 (Dense) (None, 32) 8224
dropout_11 (Dropout) (None, 32) 0
dense_9 (Dense) (None, 1) 33
=================================================================
Total params: 522,521
Trainable params: 522,521
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 4000 ['sequence_data[0][0]']
dropout_6 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_6[0][0]']
conv1d_2 (Conv1D) (None, 128, 160) 204960 ['max_pooling1d_1[0][0]']
dropout_7 (Dropout) (None, 128, 160) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 160) 0 ['dropout_7[0][0]']
conv1d_3 (Conv1D) (None, 64, 112) 286832 ['max_pooling1d_2[0][0]']
dropout_8 (Dropout) (None, 64, 112) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 112) 0 ['dropout_8[0][0]']
conv1d_4 (Conv1D) (None, 32, 8) 14344 ['max_pooling1d_3[0][0]']
dropout_9 (Dropout) (None, 32, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_4 (MaxPooling1D) (None, 16, 8) 0 ['dropout_9[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 16, 32) 4128 ['max_pooling1d_4[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_10 (Dropout) (None, 16, 32) 0 ['conv1d_5[0][0]']
dense_5 (Dense) (None, 8) 1096 ['dropout_3[0][0]']
max_pooling1d_5 (MaxPooling1D) (None, 8, 32) 0 ['dropout_10[0][0]']
dropout_4 (Dropout) (None, 8) 0 ['dense_5[0][0]']
flatten_1 (Flatten) (None, 256) 0 ['max_pooling1d_5[0][0]']
dense_6 (Dense) (None, 8) 72 ['dropout_4[0][0]']
dense_8 (Dense) (None, 32) 8224 ['flatten_1[0][0]']
dropout_5 (Dropout) (None, 8) 0 ['dense_6[0][0]']
dropout_11 (Dropout) (None, 32) 0 ['dense_8[0][0]']
concatenate (Concatenate) (None, 40) 0 ['dropout_5[0][0]',
'dropout_11[0][0]']
dense_10 (Dense) (None, 64) 2624 ['concatenate[0][0]']
dense_11 (Dense) (None, 1) 65 ['dense_10[0][0]']
==================================================================================================
Total params: 602,777
Trainable params: 602,777
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 15s 63ms/step - loss: 0.3271 - accuracy: 0.8902 - recall: 0.1311 - precision: 0.5696 - AUROC: 0.7130 - AUPRC: 0.3061 - f1_score: 0.2132 - balanced_accuracy: 0.5592 - specificity: 0.9873 - miss_rate: 0.8689 - fall_out: 0.0127 - mcc: 0.2356 - val_loss: 0.2797 - val_accuracy: 0.8933 - val_recall: 0.0627 - val_precision: 0.9474 - val_AUROC: 0.8333 - val_AUPRC: 0.5266 - val_f1_score: 0.1176 - val_balanced_accuracy: 0.5311 - val_specificity: 0.9996 - val_miss_rate: 0.9373 - val_fall_out: 4.4555e-04 - val_mcc: 0.2288
Epoch 2/1000
198/198 [==============================] - 12s 59ms/step - loss: 0.2923 - accuracy: 0.8972 - recall: 0.1764 - precision: 0.6808 - AUROC: 0.7845 - AUPRC: 0.4144 - f1_score: 0.2802 - balanced_accuracy: 0.5829 - specificity: 0.9894 - miss_rate: 0.8236 - fall_out: 0.0106 - mcc: 0.3114 - val_loss: 0.2691 - val_accuracy: 0.9050 - val_recall: 0.1875 - val_precision: 0.8791 - val_AUROC: 0.8401 - val_AUPRC: 0.5524 - val_f1_score: 0.3090 - val_balanced_accuracy: 0.5921 - val_specificity: 0.9967 - val_miss_rate: 0.8125 - val_fall_out: 0.0033 - val_mcc: 0.3801
Epoch 3/1000
198/198 [==============================] - 12s 59ms/step - loss: 0.2818 - accuracy: 0.8994 - recall: 0.1806 - precision: 0.7277 - AUROC: 0.8065 - AUPRC: 0.4431 - f1_score: 0.2894 - balanced_accuracy: 0.5860 - specificity: 0.9914 - miss_rate: 0.8194 - fall_out: 0.0086 - mcc: 0.3297 - val_loss: 0.2644 - val_accuracy: 0.9016 - val_recall: 0.1519 - val_precision: 0.8826 - val_AUROC: 0.8425 - val_AUPRC: 0.5568 - val_f1_score: 0.2592 - val_balanced_accuracy: 0.5747 - val_specificity: 0.9974 - val_miss_rate: 0.8481 - val_fall_out: 0.0026 - val_mcc: 0.3423
Epoch 4/1000
198/198 [==============================] - 12s 59ms/step - loss: 0.2765 - accuracy: 0.9003 - recall: 0.1865 - precision: 0.7407 - AUROC: 0.8177 - AUPRC: 0.4628 - f1_score: 0.2980 - balanced_accuracy: 0.5891 - specificity: 0.9916 - miss_rate: 0.8135 - fall_out: 0.0084 - mcc: 0.3392 - val_loss: 0.2661 - val_accuracy: 0.9023 - val_recall: 0.1505 - val_precision: 0.9231 - val_AUROC: 0.8464 - val_AUPRC: 0.5681 - val_f1_score: 0.2588 - val_balanced_accuracy: 0.5745 - val_specificity: 0.9984 - val_miss_rate: 0.8495 - val_fall_out: 0.0016 - val_mcc: 0.3505
Epoch 5/1000
198/198 [==============================] - 12s 60ms/step - loss: 0.2708 - accuracy: 0.8999 - recall: 0.1858 - precision: 0.7318 - AUROC: 0.8288 - AUPRC: 0.4771 - f1_score: 0.2964 - balanced_accuracy: 0.5886 - specificity: 0.9913 - miss_rate: 0.8142 - fall_out: 0.0087 - mcc: 0.3358 - val_loss: 0.2698 - val_accuracy: 0.9046 - val_recall: 0.1728 - val_precision: 0.9219 - val_AUROC: 0.8435 - val_AUPRC: 0.5716 - val_f1_score: 0.2911 - val_balanced_accuracy: 0.5855 - val_specificity: 0.9981 - val_miss_rate: 0.8272 - val_fall_out: 0.0019 - val_mcc: 0.3758
198/198 [==============================] - 7s 36ms/step - loss: 0.2615 - accuracy: 0.9059 - recall: 0.1841 - precision: 0.9305 - AUROC: 0.8682 - AUPRC: 0.6013 - f1_score: 0.3074 - balanced_accuracy: 0.5912 - specificity: 0.9982 - miss_rate: 0.8159 - fall_out: 0.0018 - mcc: 0.3904
50/50 [==============================] - 5s 99ms/step - loss: 0.2698 - accuracy: 0.9046 - recall: 0.1728 - precision: 0.9219 - AUROC: 0.8435 - AUPRC: 0.5716 - f1_score: 0.2911 - balanced_accuracy: 0.5855 - specificity: 0.9981 - miss_rate: 0.8272 - fall_out: 0.0019 - mcc: 0.3758
-- HOLDOUT 7 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['SNRNP70', 'ZNF282']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70', 'ZNF282']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 16s 68ms/step - loss: 17.3741 - accuracy: 0.8988 - recall: 0.2337 - precision: 0.6493 - AUROC: 0.7497 - AUPRC: 0.3982 - f1_score: 0.3437 - balanced_accuracy: 0.6088 - specificity: 0.9838 - miss_rate: 0.7663 - fall_out: 0.0162 - mcc: 0.3486 - val_loss: 2.5336 - val_accuracy: 0.9102 - val_recall: 0.2780 - val_precision: 0.7980 - val_AUROC: 0.8350 - val_AUPRC: 0.5719 - val_f1_score: 0.4124 - val_balanced_accuracy: 0.6345 - val_specificity: 0.9910 - val_miss_rate: 0.7220 - val_fall_out: 0.0090 - val_mcc: 0.4379
Epoch 2/1000
198/198 [==============================] - 13s 64ms/step - loss: 2.0627 - accuracy: 0.9083 - recall: 0.3041 - precision: 0.7296 - AUROC: 0.8233 - AUPRC: 0.5247 - f1_score: 0.4293 - balanced_accuracy: 0.6448 - specificity: 0.9856 - miss_rate: 0.6959 - fall_out: 0.0144 - mcc: 0.4328 - val_loss: 2.0639 - val_accuracy: 0.9053 - val_recall: 0.1868 - val_precision: 0.8963 - val_AUROC: 0.8506 - val_AUPRC: 0.5858 - val_f1_score: 0.3091 - val_balanced_accuracy: 0.5920 - val_specificity: 0.9972 - val_miss_rate: 0.8132 - val_fall_out: 0.0028 - val_mcc: 0.3841
Epoch 3/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.8364 - accuracy: 0.9093 - recall: 0.3210 - precision: 0.7267 - AUROC: 0.8384 - AUPRC: 0.5426 - f1_score: 0.4453 - balanced_accuracy: 0.6528 - specificity: 0.9846 - miss_rate: 0.6790 - fall_out: 0.0154 - mcc: 0.4441 - val_loss: 1.9567 - val_accuracy: 0.8974 - val_recall: 0.0969 - val_precision: 0.9789 - val_AUROC: 0.8545 - val_AUPRC: 0.5947 - val_f1_score: 0.1763 - val_balanced_accuracy: 0.5483 - val_specificity: 0.9997 - val_miss_rate: 0.9031 - val_fall_out: 2.6733e-04 - val_mcc: 0.2908
Epoch 4/1000
198/198 [==============================] - 13s 65ms/step - loss: 1.7096 - accuracy: 0.9111 - recall: 0.3283 - precision: 0.7454 - AUROC: 0.8465 - AUPRC: 0.5578 - f1_score: 0.4558 - balanced_accuracy: 0.6570 - specificity: 0.9857 - miss_rate: 0.6717 - fall_out: 0.0143 - mcc: 0.4570 - val_loss: 1.8493 - val_accuracy: 0.9072 - val_recall: 0.1993 - val_precision: 0.9167 - val_AUROC: 0.8584 - val_AUPRC: 0.6009 - val_f1_score: 0.3274 - val_balanced_accuracy: 0.5985 - val_specificity: 0.9977 - val_miss_rate: 0.8007 - val_fall_out: 0.0023 - val_mcc: 0.4028
Epoch 5/1000
198/198 [==============================] - 13s 65ms/step - loss: 1.6126 - accuracy: 0.9113 - recall: 0.3349 - precision: 0.7405 - AUROC: 0.8529 - AUPRC: 0.5714 - f1_score: 0.4612 - balanced_accuracy: 0.6599 - specificity: 0.9850 - miss_rate: 0.6651 - fall_out: 0.0150 - mcc: 0.4598 - val_loss: 1.8063 - val_accuracy: 0.9094 - val_recall: 0.2488 - val_precision: 0.8380 - val_AUROC: 0.8570 - val_AUPRC: 0.5982 - val_f1_score: 0.3837 - val_balanced_accuracy: 0.6213 - val_specificity: 0.9939 - val_miss_rate: 0.7512 - val_fall_out: 0.0061 - val_mcc: 0.4266
Epoch 6/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.5355 - accuracy: 0.9116 - recall: 0.3377 - precision: 0.7421 - AUROC: 0.8582 - AUPRC: 0.5790 - f1_score: 0.4642 - balanced_accuracy: 0.6613 - specificity: 0.9850 - miss_rate: 0.6623 - fall_out: 0.0150 - mcc: 0.4625 - val_loss: 1.7307 - val_accuracy: 0.9040 - val_recall: 0.1714 - val_precision: 0.9044 - val_AUROC: 0.8592 - val_AUPRC: 0.5985 - val_f1_score: 0.2882 - val_balanced_accuracy: 0.5846 - val_specificity: 0.9977 - val_miss_rate: 0.8286 - val_fall_out: 0.0023 - val_mcc: 0.3697
Epoch 7/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.4726 - accuracy: 0.9130 - recall: 0.3445 - precision: 0.7553 - AUROC: 0.8624 - AUPRC: 0.5835 - f1_score: 0.4731 - balanced_accuracy: 0.6651 - specificity: 0.9857 - miss_rate: 0.6555 - fall_out: 0.0143 - mcc: 0.4728 - val_loss: 1.6527 - val_accuracy: 0.9121 - val_recall: 0.2787 - val_precision: 0.8386 - val_AUROC: 0.8602 - val_AUPRC: 0.6046 - val_f1_score: 0.4184 - val_balanced_accuracy: 0.6359 - val_specificity: 0.9931 - val_miss_rate: 0.7213 - val_fall_out: 0.0069 - val_mcc: 0.4526
Epoch 8/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.4180 - accuracy: 0.9135 - recall: 0.3511 - precision: 0.7559 - AUROC: 0.8676 - AUPRC: 0.5945 - f1_score: 0.4795 - balanced_accuracy: 0.6683 - specificity: 0.9855 - miss_rate: 0.6489 - fall_out: 0.0145 - mcc: 0.4778 - val_loss: 1.5161 - val_accuracy: 0.9117 - val_recall: 0.2697 - val_precision: 0.8487 - val_AUROC: 0.8589 - val_AUPRC: 0.6059 - val_f1_score: 0.4093 - val_balanced_accuracy: 0.6318 - val_specificity: 0.9939 - val_miss_rate: 0.7303 - val_fall_out: 0.0061 - val_mcc: 0.4484
Epoch 9/1000
198/198 [==============================] - 13s 65ms/step - loss: 1.3727 - accuracy: 0.9138 - recall: 0.3560 - precision: 0.7542 - AUROC: 0.8689 - AUPRC: 0.5995 - f1_score: 0.4837 - balanced_accuracy: 0.6706 - specificity: 0.9852 - miss_rate: 0.6440 - fall_out: 0.0148 - mcc: 0.4806 - val_loss: 1.5597 - val_accuracy: 0.9053 - val_recall: 0.1742 - val_precision: 0.9506 - val_AUROC: 0.8598 - val_AUPRC: 0.6074 - val_f1_score: 0.2945 - val_balanced_accuracy: 0.5865 - val_specificity: 0.9988 - val_miss_rate: 0.8258 - val_fall_out: 0.0012 - val_mcc: 0.3847
Epoch 10/1000
198/198 [==============================] - 13s 65ms/step - loss: 1.3329 - accuracy: 0.9148 - recall: 0.3656 - precision: 0.7583 - AUROC: 0.8742 - AUPRC: 0.6080 - f1_score: 0.4933 - balanced_accuracy: 0.6753 - specificity: 0.9851 - miss_rate: 0.6344 - fall_out: 0.0149 - mcc: 0.4891 - val_loss: 1.5244 - val_accuracy: 0.9038 - val_recall: 0.1603 - val_precision: 0.9504 - val_AUROC: 0.8566 - val_AUPRC: 0.6052 - val_f1_score: 0.2743 - val_balanced_accuracy: 0.5796 - val_specificity: 0.9989 - val_miss_rate: 0.8397 - val_fall_out: 0.0011 - val_mcc: 0.3686
198/198 [==============================] - 8s 38ms/step - loss: 1.5091 - accuracy: 0.9035 - recall: 0.1588 - precision: 0.9431 - AUROC: 0.8947 - AUPRC: 0.6498 - f1_score: 0.2719 - balanced_accuracy: 0.5788 - specificity: 0.9988 - miss_rate: 0.8412 - fall_out: 0.0012 - mcc: 0.3651
50/50 [==============================] - 6s 109ms/step - loss: 1.5244 - accuracy: 0.9038 - recall: 0.1603 - precision: 0.9504 - AUROC: 0.8566 - AUPRC: 0.6052 - f1_score: 0.2743 - balanced_accuracy: 0.5796 - specificity: 0.9989 - miss_rate: 0.8397 - fall_out: 0.0011 - mcc: 0.3686
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_6\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_6\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 3, 'units_0': 72, 'dropout_0': 0.5, 'units_1': 8, 'dropout_1': 0.3, 'units_2': 136, 'dropout_2': 0.3, 'units_3': 136, 'dropout_3': 0.3, 'units_4': 8, 'dropout_4': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_6\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_6\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 4, 'filters_0': 160, 'kernel_size_0': 16, 'conv_dropout_0': 0.2, 'dense_depth': 1, 'units_0': 16, 'dense_dropout_0': 0.3, 'units_1': 128, 'dense_dropout_1': 0.4, 'filters_1': 8, 'kernel_size_1': 6, 'conv_dropout_1': 0.0, 'units_2': 16, 'dense_dropout_2': 0.3, 'units_3': 16, 'dense_dropout_3': 0.4, 'units_4': 64, 'dense_dropout_4': 0.3, 'filters_2': 96, 'kernel_size_2': 12, 'conv_dropout_2': 0.1, 'filters_3': 64, 'kernel_size_3': 8, 'conv_dropout_3': 0.1}
- Removing uncorrelated/correlated features: ['SNRNP70', 'ZNF282']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 72) 40464
dropout_3 (Dropout) (None, 72) 0
dense_5 (Dense) (None, 8) 584
dropout_4 (Dropout) (None, 8) 0
dense_6 (Dense) (None, 136) 1224
dropout_5 (Dropout) (None, 136) 0
dense_7 (Dense) (None, 1) 137
=================================================================
Total params: 42,409
Trainable params: 42,409
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 160) 10400
dropout_6 (Dropout) (None, 256, 160) 0
max_pooling1d_1 (MaxPooling (None, 128, 160) 0
1D)
conv1d_2 (Conv1D) (None, 128, 8) 7688
dropout_7 (Dropout) (None, 128, 8) 0
max_pooling1d_2 (MaxPooling (None, 64, 8) 0
1D)
conv1d_3 (Conv1D) (None, 64, 96) 9312
dropout_8 (Dropout) (None, 64, 96) 0
max_pooling1d_3 (MaxPooling (None, 32, 96) 0
1D)
conv1d_4 (Conv1D) (None, 32, 64) 49216
dropout_9 (Dropout) (None, 32, 64) 0
max_pooling1d_4 (MaxPooling (None, 16, 64) 0
1D)
flatten_1 (Flatten) (None, 1024) 0
dense_8 (Dense) (None, 16) 16400
dropout_10 (Dropout) (None, 16) 0
dense_9 (Dense) (None, 1) 17
=================================================================
Total params: 93,033
Trainable params: 93,033
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 160) 10400 ['sequence_data[0][0]']
dropout_6 (Dropout) (None, 256, 160) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 160) 0 ['dropout_6[0][0]']
conv1d_2 (Conv1D) (None, 128, 8) 7688 ['max_pooling1d_1[0][0]']
dropout_7 (Dropout) (None, 128, 8) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 8) 0 ['dropout_7[0][0]']
conv1d_3 (Conv1D) (None, 64, 96) 9312 ['max_pooling1d_2[0][0]']
dropout_8 (Dropout) (None, 64, 96) 0 ['conv1d_3[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 32, 96) 0 ['dropout_8[0][0]']
dense_4 (Dense) (None, 72) 40464 ['epigenomic_data[0][0]']
conv1d_4 (Conv1D) (None, 32, 64) 49216 ['max_pooling1d_3[0][0]']
dropout_3 (Dropout) (None, 72) 0 ['dense_4[0][0]']
dropout_9 (Dropout) (None, 32, 64) 0 ['conv1d_4[0][0]']
dense_5 (Dense) (None, 8) 584 ['dropout_3[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 64) 0 ['dropout_9[0][0]']
dropout_4 (Dropout) (None, 8) 0 ['dense_5[0][0]']
flatten_1 (Flatten) (None, 1024) 0 ['max_pooling1d_4[0][0]']
dense_6 (Dense) (None, 136) 1224 ['dropout_4[0][0]']
dense_8 (Dense) (None, 16) 16400 ['flatten_1[0][0]']
dropout_5 (Dropout) (None, 136) 0 ['dense_6[0][0]']
dropout_10 (Dropout) (None, 16) 0 ['dense_8[0][0]']
concatenate (Concatenate) (None, 152) 0 ['dropout_5[0][0]',
'dropout_10[0][0]']
dense_10 (Dense) (None, 64) 9792 ['concatenate[0][0]']
dense_11 (Dense) (None, 1) 65 ['dense_10[0][0]']
==================================================================================================
Total params: 145,145
Trainable params: 145,145
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 18s 61ms/step - loss: 0.3086 - accuracy: 0.8971 - recall: 0.2605 - precision: 0.6081 - AUROC: 0.7419 - AUPRC: 0.4017 - f1_score: 0.3648 - balanced_accuracy: 0.6195 - specificity: 0.9785 - miss_rate: 0.7395 - fall_out: 0.0215 - mcc: 0.3526 - val_loss: 0.2717 - val_accuracy: 0.9117 - val_recall: 0.3679 - val_precision: 0.7154 - val_AUROC: 0.8412 - val_AUPRC: 0.5751 - val_f1_score: 0.4860 - val_balanced_accuracy: 0.6746 - val_specificity: 0.9813 - val_miss_rate: 0.6321 - val_fall_out: 0.0187 - val_mcc: 0.4725
Epoch 2/1000
198/198 [==============================] - 11s 55ms/step - loss: 0.2676 - accuracy: 0.9068 - recall: 0.3279 - precision: 0.6867 - AUROC: 0.8149 - AUPRC: 0.5139 - f1_score: 0.4439 - balanced_accuracy: 0.6544 - specificity: 0.9809 - miss_rate: 0.6721 - fall_out: 0.0191 - mcc: 0.4326 - val_loss: 0.2591 - val_accuracy: 0.9136 - val_recall: 0.3889 - val_precision: 0.7209 - val_AUROC: 0.8543 - val_AUPRC: 0.5948 - val_f1_score: 0.5052 - val_balanced_accuracy: 0.6848 - val_specificity: 0.9808 - val_miss_rate: 0.6111 - val_fall_out: 0.0192 - val_mcc: 0.4891
Epoch 3/1000
198/198 [==============================] - 11s 54ms/step - loss: 0.2569 - accuracy: 0.9097 - recall: 0.3413 - precision: 0.7132 - AUROC: 0.8359 - AUPRC: 0.5435 - f1_score: 0.4617 - balanced_accuracy: 0.6619 - specificity: 0.9824 - miss_rate: 0.6587 - fall_out: 0.0176 - mcc: 0.4532 - val_loss: 0.2674 - val_accuracy: 0.9152 - val_recall: 0.4188 - val_precision: 0.7155 - val_AUROC: 0.8561 - val_AUPRC: 0.5977 - val_f1_score: 0.5284 - val_balanced_accuracy: 0.6988 - val_specificity: 0.9787 - val_miss_rate: 0.5812 - val_fall_out: 0.0213 - val_mcc: 0.5063
Epoch 4/1000
198/198 [==============================] - 11s 54ms/step - loss: 0.2502 - accuracy: 0.9116 - recall: 0.3548 - precision: 0.7257 - AUROC: 0.8456 - AUPRC: 0.5640 - f1_score: 0.4765 - balanced_accuracy: 0.6688 - specificity: 0.9828 - miss_rate: 0.6452 - fall_out: 0.0172 - mcc: 0.4678 - val_loss: 0.2623 - val_accuracy: 0.9132 - val_recall: 0.3038 - val_precision: 0.8150 - val_AUROC: 0.8591 - val_AUPRC: 0.6024 - val_f1_score: 0.4426 - val_balanced_accuracy: 0.6475 - val_specificity: 0.9912 - val_miss_rate: 0.6962 - val_fall_out: 0.0088 - val_mcc: 0.4649
198/198 [==============================] - 7s 34ms/step - loss: 0.2566 - accuracy: 0.9146 - recall: 0.3009 - precision: 0.8479 - AUROC: 0.8754 - AUPRC: 0.6196 - f1_score: 0.4442 - balanced_accuracy: 0.6470 - specificity: 0.9931 - miss_rate: 0.6991 - fall_out: 0.0069 - mcc: 0.4744
50/50 [==============================] - 5s 103ms/step - loss: 0.2623 - accuracy: 0.9132 - recall: 0.3038 - precision: 0.8150 - AUROC: 0.8591 - AUPRC: 0.6024 - f1_score: 0.4426 - balanced_accuracy: 0.6475 - specificity: 0.9912 - miss_rate: 0.6962 - fall_out: 0.0088 - mcc: 0.4649
-- HOLDOUT 8 --
-- 1 Uncorrelated features: [Pearson+Spearman]
['SNRNP70']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 18s 69ms/step - loss: 16.7425 - accuracy: 0.8983 - recall: 0.2412 - precision: 0.6359 - AUROC: 0.7466 - AUPRC: 0.3981 - f1_score: 0.3497 - balanced_accuracy: 0.6118 - specificity: 0.9823 - miss_rate: 0.7588 - fall_out: 0.0177 - mcc: 0.3494 - val_loss: 2.2878 - val_accuracy: 0.9132 - val_recall: 0.3115 - val_precision: 0.8011 - val_AUROC: 0.8425 - val_AUPRC: 0.5785 - val_f1_score: 0.4486 - val_balanced_accuracy: 0.6508 - val_specificity: 0.9901 - val_miss_rate: 0.6885 - val_fall_out: 0.0099 - val_mcc: 0.4658
Epoch 2/1000
198/198 [==============================] - 14s 66ms/step - loss: 2.0436 - accuracy: 0.9063 - recall: 0.2943 - precision: 0.7101 - AUROC: 0.8264 - AUPRC: 0.5162 - f1_score: 0.4162 - balanced_accuracy: 0.6395 - specificity: 0.9846 - miss_rate: 0.7057 - fall_out: 0.0154 - mcc: 0.4179 - val_loss: 1.7181 - val_accuracy: 0.9126 - val_recall: 0.2843 - val_precision: 0.8378 - val_AUROC: 0.8555 - val_AUPRC: 0.5970 - val_f1_score: 0.4246 - val_balanced_accuracy: 0.6386 - val_specificity: 0.9930 - val_miss_rate: 0.7157 - val_fall_out: 0.0070 - val_mcc: 0.4571
Epoch 3/1000
198/198 [==============================] - 15s 70ms/step - loss: 1.8178 - accuracy: 0.9093 - recall: 0.3178 - precision: 0.7306 - AUROC: 0.8378 - AUPRC: 0.5451 - f1_score: 0.4430 - balanced_accuracy: 0.6514 - specificity: 0.9850 - miss_rate: 0.6822 - fall_out: 0.0150 - mcc: 0.4434 - val_loss: 1.5289 - val_accuracy: 0.9125 - val_recall: 0.2704 - val_precision: 0.8661 - val_AUROC: 0.8590 - val_AUPRC: 0.6042 - val_f1_score: 0.4121 - val_balanced_accuracy: 0.6325 - val_specificity: 0.9947 - val_miss_rate: 0.7296 - val_fall_out: 0.0053 - val_mcc: 0.4548
Epoch 4/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.6944 - accuracy: 0.9113 - recall: 0.3279 - precision: 0.7493 - AUROC: 0.8470 - AUPRC: 0.5622 - f1_score: 0.4562 - balanced_accuracy: 0.6569 - specificity: 0.9860 - miss_rate: 0.6721 - fall_out: 0.0140 - mcc: 0.4583 - val_loss: 1.4203 - val_accuracy: 0.9151 - val_recall: 0.3275 - val_precision: 0.8103 - val_AUROC: 0.8608 - val_AUPRC: 0.6056 - val_f1_score: 0.4665 - val_balanced_accuracy: 0.6589 - val_specificity: 0.9902 - val_miss_rate: 0.6725 - val_fall_out: 0.0098 - val_mcc: 0.4817
Epoch 5/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.5995 - accuracy: 0.9116 - recall: 0.3337 - precision: 0.7473 - AUROC: 0.8531 - AUPRC: 0.5714 - f1_score: 0.4614 - balanced_accuracy: 0.6596 - specificity: 0.9856 - miss_rate: 0.6663 - fall_out: 0.0144 - mcc: 0.4617 - val_loss: 1.3484 - val_accuracy: 0.9159 - val_recall: 0.3366 - val_precision: 0.8118 - val_AUROC: 0.8587 - val_AUPRC: 0.6076 - val_f1_score: 0.4759 - val_balanced_accuracy: 0.6633 - val_specificity: 0.9900 - val_miss_rate: 0.6634 - val_fall_out: 0.0100 - val_mcc: 0.4892
Epoch 6/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.5247 - accuracy: 0.9130 - recall: 0.3448 - precision: 0.7554 - AUROC: 0.8576 - AUPRC: 0.5837 - f1_score: 0.4735 - balanced_accuracy: 0.6653 - specificity: 0.9857 - miss_rate: 0.6552 - fall_out: 0.0143 - mcc: 0.4731 - val_loss: 1.2735 - val_accuracy: 0.9056 - val_recall: 0.1882 - val_precision: 0.9000 - val_AUROC: 0.8623 - val_AUPRC: 0.6110 - val_f1_score: 0.3112 - val_balanced_accuracy: 0.5927 - val_specificity: 0.9973 - val_miss_rate: 0.8118 - val_fall_out: 0.0027 - val_mcc: 0.3866
Epoch 7/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.4645 - accuracy: 0.9130 - recall: 0.3394 - precision: 0.7610 - AUROC: 0.8617 - AUPRC: 0.5889 - f1_score: 0.4695 - balanced_accuracy: 0.6629 - specificity: 0.9864 - miss_rate: 0.6606 - fall_out: 0.0136 - mcc: 0.4714 - val_loss: 1.1937 - val_accuracy: 0.9149 - val_recall: 0.3101 - val_precision: 0.8365 - val_AUROC: 0.8622 - val_AUPRC: 0.6129 - val_f1_score: 0.4525 - val_balanced_accuracy: 0.6512 - val_specificity: 0.9922 - val_miss_rate: 0.6899 - val_fall_out: 0.0078 - val_mcc: 0.4777
Epoch 8/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.4137 - accuracy: 0.9134 - recall: 0.3495 - precision: 0.7562 - AUROC: 0.8668 - AUPRC: 0.5942 - f1_score: 0.4781 - balanced_accuracy: 0.6676 - specificity: 0.9856 - miss_rate: 0.6505 - fall_out: 0.0144 - mcc: 0.4768 - val_loss: 1.1606 - val_accuracy: 0.9133 - val_recall: 0.2746 - val_precision: 0.8756 - val_AUROC: 0.8635 - val_AUPRC: 0.6156 - val_f1_score: 0.4180 - val_balanced_accuracy: 0.6348 - val_specificity: 0.9950 - val_miss_rate: 0.7254 - val_fall_out: 0.0050 - val_mcc: 0.4616
Epoch 9/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.3706 - accuracy: 0.9138 - recall: 0.3532 - precision: 0.7573 - AUROC: 0.8695 - AUPRC: 0.5993 - f1_score: 0.4817 - balanced_accuracy: 0.6694 - specificity: 0.9855 - miss_rate: 0.6468 - fall_out: 0.0145 - mcc: 0.4799 - val_loss: 1.1518 - val_accuracy: 0.9097 - val_recall: 0.2321 - val_precision: 0.8904 - val_AUROC: 0.8623 - val_AUPRC: 0.6146 - val_f1_score: 0.3682 - val_balanced_accuracy: 0.6142 - val_specificity: 0.9963 - val_miss_rate: 0.7679 - val_fall_out: 0.0037 - val_mcc: 0.4276
Epoch 10/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.3327 - accuracy: 0.9148 - recall: 0.3628 - precision: 0.7602 - AUROC: 0.8719 - AUPRC: 0.6047 - f1_score: 0.4912 - balanced_accuracy: 0.6741 - specificity: 0.9854 - miss_rate: 0.6372 - fall_out: 0.0146 - mcc: 0.4879 - val_loss: 1.1318 - val_accuracy: 0.9100 - val_recall: 0.2334 - val_precision: 0.8957 - val_AUROC: 0.8622 - val_AUPRC: 0.6155 - val_f1_score: 0.3704 - val_balanced_accuracy: 0.6150 - val_specificity: 0.9965 - val_miss_rate: 0.7666 - val_fall_out: 0.0035 - val_mcc: 0.4306
Epoch 11/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.3008 - accuracy: 0.9147 - recall: 0.3645 - precision: 0.7575 - AUROC: 0.8746 - AUPRC: 0.6063 - f1_score: 0.4922 - balanced_accuracy: 0.6748 - specificity: 0.9851 - miss_rate: 0.6355 - fall_out: 0.0149 - mcc: 0.4880 - val_loss: 1.1318 - val_accuracy: 0.9066 - val_recall: 0.1930 - val_precision: 0.9203 - val_AUROC: 0.8607 - val_AUPRC: 0.6170 - val_f1_score: 0.3191 - val_balanced_accuracy: 0.5954 - val_specificity: 0.9979 - val_miss_rate: 0.8070 - val_fall_out: 0.0021 - val_mcc: 0.3972
Epoch 12/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.2705 - accuracy: 0.9148 - recall: 0.3607 - precision: 0.7639 - AUROC: 0.8785 - AUPRC: 0.6117 - f1_score: 0.4900 - balanced_accuracy: 0.6732 - specificity: 0.9857 - miss_rate: 0.6393 - fall_out: 0.0143 - mcc: 0.4880 - val_loss: 1.1215 - val_accuracy: 0.9140 - val_recall: 0.2787 - val_precision: 0.8830 - val_AUROC: 0.8598 - val_AUPRC: 0.6160 - val_f1_score: 0.4237 - val_balanced_accuracy: 0.6370 - val_specificity: 0.9953 - val_miss_rate: 0.7213 - val_fall_out: 0.0047 - val_mcc: 0.4677
Epoch 13/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.2425 - accuracy: 0.9144 - recall: 0.3622 - precision: 0.7558 - AUROC: 0.8812 - AUPRC: 0.6178 - f1_score: 0.4898 - balanced_accuracy: 0.6736 - specificity: 0.9850 - miss_rate: 0.6378 - fall_out: 0.0150 - mcc: 0.4857 - val_loss: 1.0763 - val_accuracy: 0.9091 - val_recall: 0.2237 - val_precision: 0.8966 - val_AUROC: 0.8560 - val_AUPRC: 0.6100 - val_f1_score: 0.3581 - val_balanced_accuracy: 0.6102 - val_specificity: 0.9967 - val_miss_rate: 0.7763 - val_fall_out: 0.0033 - val_mcc: 0.4215
Epoch 14/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.2179 - accuracy: 0.9149 - recall: 0.3713 - precision: 0.7534 - AUROC: 0.8841 - AUPRC: 0.6241 - f1_score: 0.4974 - balanced_accuracy: 0.6779 - specificity: 0.9844 - miss_rate: 0.6287 - fall_out: 0.0156 - mcc: 0.4911 - val_loss: 1.0825 - val_accuracy: 0.9098 - val_recall: 0.2328 - val_precision: 0.8907 - val_AUROC: 0.8586 - val_AUPRC: 0.6146 - val_f1_score: 0.3691 - val_balanced_accuracy: 0.6145 - val_specificity: 0.9963 - val_miss_rate: 0.7672 - val_fall_out: 0.0037 - val_mcc: 0.4284
Epoch 15/1000
198/198 [==============================] - 14s 65ms/step - loss: 1.1929 - accuracy: 0.9154 - recall: 0.3811 - precision: 0.7501 - AUROC: 0.8875 - AUPRC: 0.6315 - f1_score: 0.5054 - balanced_accuracy: 0.6824 - specificity: 0.9838 - miss_rate: 0.6189 - fall_out: 0.0162 - mcc: 0.4965 - val_loss: 1.0953 - val_accuracy: 0.9104 - val_recall: 0.2390 - val_precision: 0.8909 - val_AUROC: 0.8571 - val_AUPRC: 0.6135 - val_f1_score: 0.3769 - val_balanced_accuracy: 0.6176 - val_specificity: 0.9963 - val_miss_rate: 0.7610 - val_fall_out: 0.0037 - val_mcc: 0.4344
198/198 [==============================] - 8s 38ms/step - loss: 1.0672 - accuracy: 0.9114 - recall: 0.2416 - precision: 0.9131 - AUROC: 0.9128 - AUPRC: 0.6789 - f1_score: 0.3820 - balanced_accuracy: 0.6193 - specificity: 0.9971 - miss_rate: 0.7584 - fall_out: 0.0029 - mcc: 0.4435
50/50 [==============================] - 5s 105ms/step - loss: 1.0953 - accuracy: 0.9104 - recall: 0.2390 - precision: 0.8909 - AUROC: 0.8571 - AUPRC: 0.6135 - f1_score: 0.3769 - balanced_accuracy: 0.6176 - specificity: 0.9963 - miss_rate: 0.7610 - fall_out: 0.0037 - mcc: 0.4344
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_7\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_7\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 5, 'units_0': 40, 'dropout_0': 0.4, 'units_1': 8, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.3, 'units_4': 8, 'dropout_4': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_7\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_7\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 6, 'filters_0': 104, 'kernel_size_0': 6, 'conv_dropout_0': 0.1, 'dense_depth': 3, 'units_0': 80, 'dense_dropout_0': 0.3, 'units_1': 128, 'dense_dropout_1': 0.5, 'units_2': 128, 'dense_dropout_2': 0.4, 'filters_1': 40, 'kernel_size_1': 8, 'conv_dropout_1': 0.0, 'filters_2': 8, 'kernel_size_2': 6, 'conv_dropout_2': 0.1, 'filters_3': 72, 'kernel_size_3': 6, 'conv_dropout_3': 0.1, 'filters_4': 64, 'kernel_size_4': 8, 'conv_dropout_4': 0.0, 'filters_5': 8, 'kernel_size_5': 6, 'conv_dropout_5': 0.0}
- Removing uncorrelated/correlated features: ['SNRNP70']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 40) 22480
dropout_3 (Dropout) (None, 40) 0
dense_5 (Dense) (None, 8) 328
dropout_4 (Dropout) (None, 8) 0
dense_6 (Dense) (None, 8) 72
dropout_5 (Dropout) (None, 8) 0
dense_7 (Dense) (None, 8) 72
dropout_6 (Dropout) (None, 8) 0
dense_8 (Dense) (None, 8) 72
dropout_7 (Dropout) (None, 8) 0
dense_9 (Dense) (None, 1) 9
=================================================================
Total params: 23,033
Trainable params: 23,033
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 104) 2600
dropout_8 (Dropout) (None, 256, 104) 0
max_pooling1d_1 (MaxPooling (None, 128, 104) 0
1D)
conv1d_2 (Conv1D) (None, 128, 40) 33320
dropout_9 (Dropout) (None, 128, 40) 0
max_pooling1d_2 (MaxPooling (None, 64, 40) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 1928
dropout_10 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
conv1d_4 (Conv1D) (None, 32, 72) 3528
dropout_11 (Dropout) (None, 32, 72) 0
max_pooling1d_4 (MaxPooling (None, 16, 72) 0
1D)
conv1d_5 (Conv1D) (None, 16, 64) 36928
dropout_12 (Dropout) (None, 16, 64) 0
max_pooling1d_5 (MaxPooling (None, 8, 64) 0
1D)
conv1d_6 (Conv1D) (None, 8, 8) 3080
dropout_13 (Dropout) (None, 8, 8) 0
max_pooling1d_6 (MaxPooling (None, 4, 8) 0
1D)
flatten_1 (Flatten) (None, 32) 0
dense_10 (Dense) (None, 80) 2640
dropout_14 (Dropout) (None, 80) 0
dense_11 (Dense) (None, 128) 10368
dropout_15 (Dropout) (None, 128) 0
dense_12 (Dense) (None, 128) 16512
dropout_16 (Dropout) (None, 128) 0
dense_13 (Dense) (None, 1) 129
=================================================================
Total params: 111,033
Trainable params: 111,033
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 104) 2600 ['sequence_data[0][0]']
dropout_8 (Dropout) (None, 256, 104) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 104) 0 ['dropout_8[0][0]']
conv1d_2 (Conv1D) (None, 128, 40) 33320 ['max_pooling1d_1[0][0]']
dropout_9 (Dropout) (None, 128, 40) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 40) 0 ['dropout_9[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 1928 ['max_pooling1d_2[0][0]']
dropout_10 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_10[0][0]']
conv1d_4 (Conv1D) (None, 32, 72) 3528 ['max_pooling1d_3[0][0]']
dropout_11 (Dropout) (None, 32, 72) 0 ['conv1d_4[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 16, 72) 0 ['dropout_11[0][0]']
conv1d_5 (Conv1D) (None, 16, 64) 36928 ['max_pooling1d_4[0][0]']
dropout_12 (Dropout) (None, 16, 64) 0 ['conv1d_5[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_5 (MaxPooling1D) (None, 8, 64) 0 ['dropout_12[0][0]']
dense_4 (Dense) (None, 40) 22480 ['epigenomic_data[0][0]']
conv1d_6 (Conv1D) (None, 8, 8) 3080 ['max_pooling1d_5[0][0]']
dropout_3 (Dropout) (None, 40) 0 ['dense_4[0][0]']
dropout_13 (Dropout) (None, 8, 8) 0 ['conv1d_6[0][0]']
dense_5 (Dense) (None, 8) 328 ['dropout_3[0][0]']
max_pooling1d_6 (MaxPooling1D) (None, 4, 8) 0 ['dropout_13[0][0]']
dropout_4 (Dropout) (None, 8) 0 ['dense_5[0][0]']
flatten_1 (Flatten) (None, 32) 0 ['max_pooling1d_6[0][0]']
dense_6 (Dense) (None, 8) 72 ['dropout_4[0][0]']
dense_10 (Dense) (None, 80) 2640 ['flatten_1[0][0]']
dropout_5 (Dropout) (None, 8) 0 ['dense_6[0][0]']
dropout_14 (Dropout) (None, 80) 0 ['dense_10[0][0]']
dense_7 (Dense) (None, 8) 72 ['dropout_5[0][0]']
dense_11 (Dense) (None, 128) 10368 ['dropout_14[0][0]']
dropout_6 (Dropout) (None, 8) 0 ['dense_7[0][0]']
dropout_15 (Dropout) (None, 128) 0 ['dense_11[0][0]']
dense_8 (Dense) (None, 8) 72 ['dropout_6[0][0]']
dense_12 (Dense) (None, 128) 16512 ['dropout_15[0][0]']
dropout_7 (Dropout) (None, 8) 0 ['dense_8[0][0]']
dropout_16 (Dropout) (None, 128) 0 ['dense_12[0][0]']
concatenate (Concatenate) (None, 136) 0 ['dropout_7[0][0]',
'dropout_16[0][0]']
dense_14 (Dense) (None, 64) 8768 ['concatenate[0][0]']
dense_15 (Dense) (None, 1) 65 ['dense_14[0][0]']
==================================================================================================
Total params: 142,761
Trainable params: 142,761
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 18s 68ms/step - loss: 0.3515 - accuracy: 0.8867 - recall: 0.0106 - precision: 0.5304 - AUROC: 0.6209 - AUPRC: 0.1769 - f1_score: 0.0208 - balanced_accuracy: 0.5047 - specificity: 0.9988 - miss_rate: 0.9894 - fall_out: 0.0012 - mcc: 0.0627 - val_loss: 0.2972 - val_accuracy: 0.8892 - val_recall: 0.0237 - val_precision: 0.9714 - val_AUROC: 0.8192 - val_AUPRC: 0.5124 - val_f1_score: 0.0463 - val_balanced_accuracy: 0.5118 - val_specificity: 0.9999 - val_miss_rate: 0.9763 - val_fall_out: 8.9111e-05 - val_mcc: 0.1425
Epoch 2/1000
198/198 [==============================] - 14s 65ms/step - loss: 0.3073 - accuracy: 0.8908 - recall: 0.0634 - precision: 0.7054 - AUROC: 0.7588 - AUPRC: 0.3509 - f1_score: 0.1163 - balanced_accuracy: 0.5300 - specificity: 0.9966 - miss_rate: 0.9366 - fall_out: 0.0034 - mcc: 0.1894 - val_loss: 0.3012 - val_accuracy: 0.8978 - val_recall: 0.1108 - val_precision: 0.9034 - val_AUROC: 0.8297 - val_AUPRC: 0.5610 - val_f1_score: 0.1974 - val_balanced_accuracy: 0.5546 - val_specificity: 0.9985 - val_miss_rate: 0.8892 - val_fall_out: 0.0015 - val_mcc: 0.2959
Epoch 3/1000
198/198 [==============================] - 15s 75ms/step - loss: 0.2882 - accuracy: 0.8924 - recall: 0.0913 - precision: 0.6959 - AUROC: 0.8018 - AUPRC: 0.4097 - f1_score: 0.1614 - balanced_accuracy: 0.5431 - specificity: 0.9949 - miss_rate: 0.9087 - fall_out: 0.0051 - mcc: 0.2257 - val_loss: 0.2593 - val_accuracy: 0.9029 - val_recall: 0.1819 - val_precision: 0.8259 - val_AUROC: 0.8463 - val_AUPRC: 0.5613 - val_f1_score: 0.2981 - val_balanced_accuracy: 0.5885 - val_specificity: 0.9951 - val_miss_rate: 0.8181 - val_fall_out: 0.0049 - val_mcc: 0.3596
Epoch 4/1000
198/198 [==============================] - 15s 75ms/step - loss: 0.2773 - accuracy: 0.8937 - recall: 0.1221 - precision: 0.6740 - AUROC: 0.8172 - AUPRC: 0.4447 - f1_score: 0.2067 - balanced_accuracy: 0.5573 - specificity: 0.9924 - miss_rate: 0.8779 - fall_out: 0.0076 - mcc: 0.2560 - val_loss: 0.2530 - val_accuracy: 0.9107 - val_recall: 0.2767 - val_precision: 0.8119 - val_AUROC: 0.8498 - val_AUPRC: 0.5794 - val_f1_score: 0.4127 - val_balanced_accuracy: 0.6342 - val_specificity: 0.9918 - val_miss_rate: 0.7233 - val_fall_out: 0.0082 - val_mcc: 0.4416
Epoch 5/1000
198/198 [==============================] - 15s 72ms/step - loss: 0.2701 - accuracy: 0.8980 - recall: 0.1851 - precision: 0.6862 - AUROC: 0.8245 - AUPRC: 0.4753 - f1_score: 0.2916 - balanced_accuracy: 0.5871 - specificity: 0.9892 - miss_rate: 0.8149 - fall_out: 0.0108 - mcc: 0.3209 - val_loss: 0.2517 - val_accuracy: 0.9133 - val_recall: 0.3408 - val_precision: 0.7641 - val_AUROC: 0.8542 - val_AUPRC: 0.5862 - val_f1_score: 0.4713 - val_balanced_accuracy: 0.6637 - val_specificity: 0.9865 - val_miss_rate: 0.6592 - val_fall_out: 0.0135 - val_mcc: 0.4736
Epoch 6/1000
198/198 [==============================] - 14s 65ms/step - loss: 0.2646 - accuracy: 0.8995 - recall: 0.2173 - precision: 0.6783 - AUROC: 0.8340 - AUPRC: 0.4895 - f1_score: 0.3292 - balanced_accuracy: 0.6021 - specificity: 0.9868 - miss_rate: 0.7827 - fall_out: 0.0132 - mcc: 0.3459 - val_loss: 0.2555 - val_accuracy: 0.9105 - val_recall: 0.2913 - val_precision: 0.7828 - val_AUROC: 0.8452 - val_AUPRC: 0.5756 - val_f1_score: 0.4246 - val_balanced_accuracy: 0.6405 - val_specificity: 0.9897 - val_miss_rate: 0.7087 - val_fall_out: 0.0103 - val_mcc: 0.4431
Epoch 7/1000
198/198 [==============================] - 13s 65ms/step - loss: 0.2628 - accuracy: 0.8985 - recall: 0.2090 - precision: 0.6678 - AUROC: 0.8363 - AUPRC: 0.4903 - f1_score: 0.3183 - balanced_accuracy: 0.5978 - specificity: 0.9867 - miss_rate: 0.7910 - fall_out: 0.0133 - mcc: 0.3354 - val_loss: 0.2522 - val_accuracy: 0.9047 - val_recall: 0.2418 - val_precision: 0.7462 - val_AUROC: 0.8477 - val_AUPRC: 0.5535 - val_f1_score: 0.3653 - val_balanced_accuracy: 0.6156 - val_specificity: 0.9895 - val_miss_rate: 0.7582 - val_fall_out: 0.0105 - val_mcc: 0.3898
198/198 [==============================] - 7s 35ms/step - loss: 0.2441 - accuracy: 0.9062 - recall: 0.2536 - precision: 0.7595 - AUROC: 0.8673 - AUPRC: 0.5738 - f1_score: 0.3802 - balanced_accuracy: 0.6216 - specificity: 0.9897 - miss_rate: 0.7464 - fall_out: 0.0103 - mcc: 0.4042
50/50 [==============================] - 5s 103ms/step - loss: 0.2522 - accuracy: 0.9047 - recall: 0.2418 - precision: 0.7462 - AUROC: 0.8477 - AUPRC: 0.5535 - f1_score: 0.3653 - balanced_accuracy: 0.6156 - specificity: 0.9895 - miss_rate: 0.7582 - fall_out: 0.0105 - mcc: 0.3898
-- HOLDOUT 9 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['SNRNP70', 'ATM']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70', 'ATM']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 18s 70ms/step - loss: 17.2474 - accuracy: 0.8948 - recall: 0.2388 - precision: 0.5899 - AUROC: 0.7373 - AUPRC: 0.3787 - f1_score: 0.3399 - balanced_accuracy: 0.6088 - specificity: 0.9788 - miss_rate: 0.7612 - fall_out: 0.0212 - mcc: 0.3296 - val_loss: 2.2181 - val_accuracy: 0.9132 - val_recall: 0.3038 - val_precision: 0.8134 - val_AUROC: 0.8442 - val_AUPRC: 0.5906 - val_f1_score: 0.4424 - val_balanced_accuracy: 0.6475 - val_specificity: 0.9911 - val_miss_rate: 0.6962 - val_fall_out: 0.0089 - val_mcc: 0.4643
Epoch 2/1000
198/198 [==============================] - 14s 66ms/step - loss: 2.0401 - accuracy: 0.9075 - recall: 0.3051 - precision: 0.7166 - AUROC: 0.8219 - AUPRC: 0.5189 - f1_score: 0.4280 - balanced_accuracy: 0.6448 - specificity: 0.9846 - miss_rate: 0.6949 - fall_out: 0.0154 - mcc: 0.4285 - val_loss: 1.6874 - val_accuracy: 0.9085 - val_recall: 0.2216 - val_precision: 0.8858 - val_AUROC: 0.8546 - val_AUPRC: 0.5991 - val_f1_score: 0.3545 - val_balanced_accuracy: 0.6090 - val_specificity: 0.9963 - val_miss_rate: 0.7784 - val_fall_out: 0.0037 - val_mcc: 0.4162
Epoch 3/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.8173 - accuracy: 0.9085 - recall: 0.3116 - precision: 0.7255 - AUROC: 0.8389 - AUPRC: 0.5415 - f1_score: 0.4359 - balanced_accuracy: 0.6482 - specificity: 0.9849 - miss_rate: 0.6884 - fall_out: 0.0151 - mcc: 0.4368 - val_loss: 1.5480 - val_accuracy: 0.9068 - val_recall: 0.1937 - val_precision: 0.9267 - val_AUROC: 0.8588 - val_AUPRC: 0.6088 - val_f1_score: 0.3205 - val_balanced_accuracy: 0.5959 - val_specificity: 0.9980 - val_miss_rate: 0.8063 - val_fall_out: 0.0020 - val_mcc: 0.3997
Epoch 4/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.6971 - accuracy: 0.9111 - recall: 0.3319 - precision: 0.7419 - AUROC: 0.8443 - AUPRC: 0.5589 - f1_score: 0.4587 - balanced_accuracy: 0.6586 - specificity: 0.9852 - miss_rate: 0.6681 - fall_out: 0.0148 - mcc: 0.4583 - val_loss: 1.3883 - val_accuracy: 0.9044 - val_recall: 0.1666 - val_precision: 0.9447 - val_AUROC: 0.8603 - val_AUPRC: 0.6125 - val_f1_score: 0.2832 - val_balanced_accuracy: 0.5827 - val_specificity: 0.9988 - val_miss_rate: 0.8334 - val_fall_out: 0.0012 - val_mcc: 0.3745
Epoch 5/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.6029 - accuracy: 0.9115 - recall: 0.3347 - precision: 0.7438 - AUROC: 0.8544 - AUPRC: 0.5726 - f1_score: 0.4617 - balanced_accuracy: 0.6600 - specificity: 0.9853 - miss_rate: 0.6653 - fall_out: 0.0147 - mcc: 0.4610 - val_loss: 1.3558 - val_accuracy: 0.8959 - val_recall: 0.0843 - val_precision: 0.9758 - val_AUROC: 0.8602 - val_AUPRC: 0.6111 - val_f1_score: 0.1552 - val_balanced_accuracy: 0.5420 - val_specificity: 0.9997 - val_miss_rate: 0.9157 - val_fall_out: 2.6733e-04 - val_mcc: 0.2706
Epoch 6/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.5305 - accuracy: 0.9125 - recall: 0.3438 - precision: 0.7489 - AUROC: 0.8577 - AUPRC: 0.5790 - f1_score: 0.4712 - balanced_accuracy: 0.6645 - specificity: 0.9853 - miss_rate: 0.6562 - fall_out: 0.0147 - mcc: 0.4696 - val_loss: 1.2791 - val_accuracy: 0.9141 - val_recall: 0.2913 - val_precision: 0.8566 - val_AUROC: 0.8622 - val_AUPRC: 0.6153 - val_f1_score: 0.4347 - val_balanced_accuracy: 0.6425 - val_specificity: 0.9938 - val_miss_rate: 0.7087 - val_fall_out: 0.0062 - val_mcc: 0.4694
Epoch 7/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.4715 - accuracy: 0.9137 - recall: 0.3516 - precision: 0.7570 - AUROC: 0.8587 - AUPRC: 0.5877 - f1_score: 0.4802 - balanced_accuracy: 0.6686 - specificity: 0.9856 - miss_rate: 0.6484 - fall_out: 0.0144 - mcc: 0.4786 - val_loss: 1.2091 - val_accuracy: 0.9069 - val_recall: 0.2000 - val_precision: 0.9054 - val_AUROC: 0.8611 - val_AUPRC: 0.6130 - val_f1_score: 0.3276 - val_balanced_accuracy: 0.5987 - val_specificity: 0.9973 - val_miss_rate: 0.8000 - val_fall_out: 0.0027 - val_mcc: 0.4004
Epoch 8/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.4186 - accuracy: 0.9133 - recall: 0.3490 - precision: 0.7551 - AUROC: 0.8646 - AUPRC: 0.5918 - f1_score: 0.4774 - balanced_accuracy: 0.6673 - specificity: 0.9855 - miss_rate: 0.6510 - fall_out: 0.0145 - mcc: 0.4760 - val_loss: 1.1702 - val_accuracy: 0.9067 - val_recall: 0.1923 - val_precision: 0.9262 - val_AUROC: 0.8605 - val_AUPRC: 0.6129 - val_f1_score: 0.3185 - val_balanced_accuracy: 0.5952 - val_specificity: 0.9980 - val_miss_rate: 0.8077 - val_fall_out: 0.0020 - val_mcc: 0.3981
Epoch 9/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.3744 - accuracy: 0.9139 - recall: 0.3495 - precision: 0.7625 - AUROC: 0.8673 - AUPRC: 0.5964 - f1_score: 0.4793 - balanced_accuracy: 0.6678 - specificity: 0.9861 - miss_rate: 0.6505 - fall_out: 0.0139 - mcc: 0.4794 - val_loss: 1.1781 - val_accuracy: 0.9167 - val_recall: 0.3289 - val_precision: 0.8384 - val_AUROC: 0.8599 - val_AUPRC: 0.6129 - val_f1_score: 0.4725 - val_balanced_accuracy: 0.6604 - val_specificity: 0.9919 - val_miss_rate: 0.6711 - val_fall_out: 0.0081 - val_mcc: 0.4934
Epoch 10/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.3351 - accuracy: 0.9150 - recall: 0.3617 - precision: 0.7644 - AUROC: 0.8700 - AUPRC: 0.6071 - f1_score: 0.4911 - balanced_accuracy: 0.6737 - specificity: 0.9857 - miss_rate: 0.6383 - fall_out: 0.0143 - mcc: 0.4889 - val_loss: 1.1245 - val_accuracy: 0.9145 - val_recall: 0.2927 - val_precision: 0.8624 - val_AUROC: 0.8601 - val_AUPRC: 0.6133 - val_f1_score: 0.4370 - val_balanced_accuracy: 0.6434 - val_specificity: 0.9940 - val_miss_rate: 0.7073 - val_fall_out: 0.0060 - val_mcc: 0.4726
Epoch 11/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.3009 - accuracy: 0.9145 - recall: 0.3617 - precision: 0.7580 - AUROC: 0.8747 - AUPRC: 0.6069 - f1_score: 0.4897 - balanced_accuracy: 0.6735 - specificity: 0.9852 - miss_rate: 0.6383 - fall_out: 0.0148 - mcc: 0.4863 - val_loss: 1.0837 - val_accuracy: 0.9151 - val_recall: 0.3003 - val_precision: 0.8603 - val_AUROC: 0.8591 - val_AUPRC: 0.6118 - val_f1_score: 0.4452 - val_balanced_accuracy: 0.6471 - val_specificity: 0.9938 - val_miss_rate: 0.6997 - val_fall_out: 0.0062 - val_mcc: 0.4783
Epoch 12/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.2702 - accuracy: 0.9156 - recall: 0.3683 - precision: 0.7666 - AUROC: 0.8797 - AUPRC: 0.6163 - f1_score: 0.4976 - balanced_accuracy: 0.6770 - specificity: 0.9857 - miss_rate: 0.6317 - fall_out: 0.0143 - mcc: 0.4945 - val_loss: 1.0638 - val_accuracy: 0.9170 - val_recall: 0.3401 - val_precision: 0.8257 - val_AUROC: 0.8585 - val_AUPRC: 0.6151 - val_f1_score: 0.4817 - val_balanced_accuracy: 0.6654 - val_specificity: 0.9908 - val_miss_rate: 0.6599 - val_fall_out: 0.0092 - val_mcc: 0.4972
Epoch 13/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.2417 - accuracy: 0.9151 - recall: 0.3743 - precision: 0.7530 - AUROC: 0.8820 - AUPRC: 0.6229 - f1_score: 0.5000 - balanced_accuracy: 0.6793 - specificity: 0.9843 - miss_rate: 0.6257 - fall_out: 0.0157 - mcc: 0.4930 - val_loss: 1.0725 - val_accuracy: 0.9116 - val_recall: 0.2453 - val_precision: 0.9072 - val_AUROC: 0.8596 - val_AUPRC: 0.6140 - val_f1_score: 0.3862 - val_balanced_accuracy: 0.6210 - val_specificity: 0.9968 - val_miss_rate: 0.7547 - val_fall_out: 0.0032 - val_mcc: 0.4453
Epoch 14/1000
198/198 [==============================] - 14s 66ms/step - loss: 1.2158 - accuracy: 0.9160 - recall: 0.3824 - precision: 0.7570 - AUROC: 0.8843 - AUPRC: 0.6270 - f1_score: 0.5082 - balanced_accuracy: 0.6834 - specificity: 0.9843 - miss_rate: 0.6176 - fall_out: 0.0157 - mcc: 0.5004 - val_loss: 1.0642 - val_accuracy: 0.9117 - val_recall: 0.2488 - val_precision: 0.9015 - val_AUROC: 0.8562 - val_AUPRC: 0.6101 - val_f1_score: 0.3900 - val_balanced_accuracy: 0.6227 - val_specificity: 0.9965 - val_miss_rate: 0.7512 - val_fall_out: 0.0035 - val_mcc: 0.4467
198/198 [==============================] - 8s 39ms/step - loss: 1.0374 - accuracy: 0.9121 - recall: 0.2529 - precision: 0.9019 - AUROC: 0.9073 - AUPRC: 0.6683 - f1_score: 0.3950 - balanced_accuracy: 0.6247 - specificity: 0.9965 - miss_rate: 0.7471 - fall_out: 0.0035 - mcc: 0.4506
50/50 [==============================] - 6s 108ms/step - loss: 1.0642 - accuracy: 0.9117 - recall: 0.2488 - precision: 0.9015 - AUROC: 0.8562 - AUPRC: 0.6101 - f1_score: 0.3900 - balanced_accuracy: 0.6227 - specificity: 0.9965 - miss_rate: 0.7512 - fall_out: 0.0035 - mcc: 0.4467
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_8\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_8\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 2, 'units_0': 136, 'dropout_0': 0.5, 'units_1': 8, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.3, 'units_3': 8, 'dropout_3': 0.3, 'units_4': 8, 'dropout_4': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_8\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_8\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 3, 'filters_0': 112, 'kernel_size_0': 6, 'conv_dropout_0': 0.1, 'dense_depth': 1, 'units_0': 32, 'dense_dropout_0': 0.5, 'units_1': 112, 'dense_dropout_1': 0.4, 'units_2': 48, 'dense_dropout_2': 0.5, 'filters_1': 8, 'kernel_size_1': 6, 'conv_dropout_1': 0.0, 'filters_2': 8, 'kernel_size_2': 6, 'conv_dropout_2': 0.0}
- Removing uncorrelated/correlated features: ['SNRNP70', 'ATM']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 8) 1096
dropout_4 (Dropout) (None, 8) 0
dense_6 (Dense) (None, 1) 9
=================================================================
Total params: 77,537
Trainable params: 77,537
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 112) 2800
dropout_5 (Dropout) (None, 256, 112) 0
max_pooling1d_1 (MaxPooling (None, 128, 112) 0
1D)
conv1d_2 (Conv1D) (None, 128, 8) 5384
dropout_6 (Dropout) (None, 128, 8) 0
max_pooling1d_2 (MaxPooling (None, 64, 8) 0
1D)
conv1d_3 (Conv1D) (None, 64, 8) 392
dropout_7 (Dropout) (None, 64, 8) 0
max_pooling1d_3 (MaxPooling (None, 32, 8) 0
1D)
flatten_1 (Flatten) (None, 256) 0
dense_7 (Dense) (None, 32) 8224
dropout_8 (Dropout) (None, 32) 0
dense_8 (Dense) (None, 1) 33
=================================================================
Total params: 16,833
Trainable params: 16,833
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 112) 2800 ['sequence_data[0][0]']
dropout_5 (Dropout) (None, 256, 112) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 112) 0 ['dropout_5[0][0]']
conv1d_2 (Conv1D) (None, 128, 8) 5384 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 128, 8) 0 ['conv1d_2[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 64, 8) 0 ['dropout_6[0][0]']
conv1d_3 (Conv1D) (None, 64, 8) 392 ['max_pooling1d_2[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
dropout_7 (Dropout) (None, 64, 8) 0 ['conv1d_3[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 8) 0 ['dropout_7[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
flatten_1 (Flatten) (None, 256) 0 ['max_pooling1d_3[0][0]']
dense_5 (Dense) (None, 8) 1096 ['dropout_3[0][0]']
dense_7 (Dense) (None, 32) 8224 ['flatten_1[0][0]']
dropout_4 (Dropout) (None, 8) 0 ['dense_5[0][0]']
dropout_8 (Dropout) (None, 32) 0 ['dense_7[0][0]']
concatenate (Concatenate) (None, 40) 0 ['dropout_4[0][0]',
'dropout_8[0][0]']
dense_9 (Dense) (None, 64) 2624 ['concatenate[0][0]']
dense_10 (Dense) (None, 1) 65 ['dense_9[0][0]']
==================================================================================================
Total params: 97,017
Trainable params: 97,017
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 15s 58ms/step - loss: 0.2992 - accuracy: 0.9009 - recall: 0.2621 - precision: 0.6581 - AUROC: 0.7642 - AUPRC: 0.4346 - f1_score: 0.3749 - balanced_accuracy: 0.6223 - specificity: 0.9826 - miss_rate: 0.7379 - fall_out: 0.0174 - mcc: 0.3736 - val_loss: 0.2492 - val_accuracy: 0.9102 - val_recall: 0.2397 - val_precision: 0.8843 - val_AUROC: 0.8519 - val_AUPRC: 0.5957 - val_f1_score: 0.3772 - val_balanced_accuracy: 0.6179 - val_specificity: 0.9960 - val_miss_rate: 0.7603 - val_fall_out: 0.0040 - val_mcc: 0.4330
Epoch 2/1000
198/198 [==============================] - 10s 50ms/step - loss: 0.2635 - accuracy: 0.9095 - recall: 0.3145 - precision: 0.7362 - AUROC: 0.8237 - AUPRC: 0.5318 - f1_score: 0.4408 - balanced_accuracy: 0.6501 - specificity: 0.9856 - miss_rate: 0.6855 - fall_out: 0.0144 - mcc: 0.4432 - val_loss: 0.2509 - val_accuracy: 0.9153 - val_recall: 0.3282 - val_precision: 0.8135 - val_AUROC: 0.8564 - val_AUPRC: 0.6048 - val_f1_score: 0.4677 - val_balanced_accuracy: 0.6593 - val_specificity: 0.9904 - val_miss_rate: 0.6718 - val_fall_out: 0.0096 - val_mcc: 0.4835
Epoch 3/1000
198/198 [==============================] - 10s 50ms/step - loss: 0.2548 - accuracy: 0.9110 - recall: 0.3274 - precision: 0.7443 - AUROC: 0.8394 - AUPRC: 0.5528 - f1_score: 0.4548 - balanced_accuracy: 0.6565 - specificity: 0.9856 - miss_rate: 0.6726 - fall_out: 0.0144 - mcc: 0.4559 - val_loss: 0.2406 - val_accuracy: 0.9148 - val_recall: 0.3045 - val_precision: 0.8436 - val_AUROC: 0.8605 - val_AUPRC: 0.6107 - val_f1_score: 0.4475 - val_balanced_accuracy: 0.6487 - val_specificity: 0.9928 - val_miss_rate: 0.6955 - val_fall_out: 0.0072 - val_mcc: 0.4758
Epoch 4/1000
198/198 [==============================] - 10s 50ms/step - loss: 0.2478 - accuracy: 0.9129 - recall: 0.3393 - precision: 0.7600 - AUROC: 0.8506 - AUPRC: 0.5710 - f1_score: 0.4691 - balanced_accuracy: 0.6628 - specificity: 0.9863 - miss_rate: 0.6607 - fall_out: 0.0137 - mcc: 0.4709 - val_loss: 0.2402 - val_accuracy: 0.9125 - val_recall: 0.2662 - val_precision: 0.8761 - val_AUROC: 0.8608 - val_AUPRC: 0.6139 - val_f1_score: 0.4083 - val_balanced_accuracy: 0.6307 - val_specificity: 0.9952 - val_miss_rate: 0.7338 - val_fall_out: 0.0048 - val_mcc: 0.4544
Epoch 5/1000
198/198 [==============================] - 10s 51ms/step - loss: 0.2433 - accuracy: 0.9135 - recall: 0.3398 - precision: 0.7678 - AUROC: 0.8587 - AUPRC: 0.5812 - f1_score: 0.4711 - balanced_accuracy: 0.6633 - specificity: 0.9869 - miss_rate: 0.6602 - fall_out: 0.0131 - mcc: 0.4744 - val_loss: 0.2403 - val_accuracy: 0.9145 - val_recall: 0.3115 - val_precision: 0.8262 - val_AUROC: 0.8605 - val_AUPRC: 0.6098 - val_f1_score: 0.4524 - val_balanced_accuracy: 0.6516 - val_specificity: 0.9916 - val_miss_rate: 0.6885 - val_fall_out: 0.0084 - val_mcc: 0.4751
Epoch 6/1000
198/198 [==============================] - 10s 50ms/step - loss: 0.2395 - accuracy: 0.9134 - recall: 0.3408 - precision: 0.7662 - AUROC: 0.8650 - AUPRC: 0.5901 - f1_score: 0.4718 - balanced_accuracy: 0.6638 - specificity: 0.9867 - miss_rate: 0.6592 - fall_out: 0.0133 - mcc: 0.4745 - val_loss: 0.2368 - val_accuracy: 0.9168 - val_recall: 0.3596 - val_precision: 0.7938 - val_AUROC: 0.8591 - val_AUPRC: 0.6146 - val_f1_score: 0.4950 - val_balanced_accuracy: 0.6738 - val_specificity: 0.9881 - val_miss_rate: 0.6404 - val_fall_out: 0.0119 - val_mcc: 0.4994
Epoch 7/1000
198/198 [==============================] - 10s 50ms/step - loss: 0.2377 - accuracy: 0.9140 - recall: 0.3419 - precision: 0.7741 - AUROC: 0.8678 - AUPRC: 0.5951 - f1_score: 0.4743 - balanced_accuracy: 0.6646 - specificity: 0.9872 - miss_rate: 0.6581 - fall_out: 0.0128 - mcc: 0.4784 - val_loss: 0.2357 - val_accuracy: 0.9167 - val_recall: 0.3352 - val_precision: 0.8279 - val_AUROC: 0.8614 - val_AUPRC: 0.6157 - val_f1_score: 0.4772 - val_balanced_accuracy: 0.6631 - val_specificity: 0.9911 - val_miss_rate: 0.6648 - val_fall_out: 0.0089 - val_mcc: 0.4943
Epoch 8/1000
198/198 [==============================] - 10s 50ms/step - loss: 0.2342 - accuracy: 0.9144 - recall: 0.3478 - precision: 0.7725 - AUROC: 0.8732 - AUPRC: 0.6032 - f1_score: 0.4796 - balanced_accuracy: 0.6673 - specificity: 0.9869 - miss_rate: 0.6522 - fall_out: 0.0131 - mcc: 0.4822 - val_loss: 0.2395 - val_accuracy: 0.9161 - val_recall: 0.3192 - val_precision: 0.8435 - val_AUROC: 0.8586 - val_AUPRC: 0.6130 - val_f1_score: 0.4631 - val_balanced_accuracy: 0.6558 - val_specificity: 0.9924 - val_miss_rate: 0.6808 - val_fall_out: 0.0076 - val_mcc: 0.4875
Epoch 9/1000
198/198 [==============================] - 10s 51ms/step - loss: 0.2312 - accuracy: 0.9158 - recall: 0.3608 - precision: 0.7781 - AUROC: 0.8782 - AUPRC: 0.6110 - f1_score: 0.4930 - balanced_accuracy: 0.6738 - specificity: 0.9868 - miss_rate: 0.6392 - fall_out: 0.0132 - mcc: 0.4939 - val_loss: 0.2409 - val_accuracy: 0.9162 - val_recall: 0.3296 - val_precision: 0.8269 - val_AUROC: 0.8582 - val_AUPRC: 0.6116 - val_f1_score: 0.4714 - val_balanced_accuracy: 0.6604 - val_specificity: 0.9912 - val_miss_rate: 0.6704 - val_fall_out: 0.0088 - val_mcc: 0.4896
198/198 [==============================] - 7s 34ms/step - loss: 0.2210 - accuracy: 0.9162 - recall: 0.3248 - precision: 0.8367 - AUROC: 0.9034 - AUPRC: 0.6550 - f1_score: 0.4679 - balanced_accuracy: 0.6583 - specificity: 0.9919 - miss_rate: 0.6752 - fall_out: 0.0081 - mcc: 0.4895
50/50 [==============================] - 6s 113ms/step - loss: 0.2409 - accuracy: 0.9162 - recall: 0.3296 - precision: 0.8269 - AUROC: 0.8582 - AUPRC: 0.6116 - f1_score: 0.4714 - balanced_accuracy: 0.6604 - specificity: 0.9912 - miss_rate: 0.6704 - fall_out: 0.0088 - mcc: 0.4896
-- HOLDOUT 10 --
-- 2 Uncorrelated features: [Pearson+Spearman]
['SNRNP70', 'ZNF282']
-- Actually uncorrelated features: [confirmed via MIC]
['SNRNP70', 'ZNF282']
-- 0 Correlated features: [Pearson]
[]
- Building Fixed MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense (Dense) (None, 128) 71936
dropout (Dropout) (None, 128) 0
dense_1 (Dense) (None, 64) 8256
dropout_1 (Dropout) (None, 64) 0
dense_2 (Dense) (None, 64) 4160
dropout_2 (Dropout) (None, 64) 0
dense_3 (Dense) (None, 1) 65
=================================================================
Total params: 84,417
Trainable params: 84,417
Non-trainable params: 0
_________________________________________________________________
- Building Fixed CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d (Conv1D) (None, 249, 160) 5280
dropout_3 (Dropout) (None, 249, 160) 0
conv1d_1 (Conv1D) (None, 249, 64) 81984
dropout_4 (Dropout) (None, 249, 64) 0
max_pooling1d (MaxPooling1D (None, 124, 64) 0
)
conv1d_2 (Conv1D) (None, 124, 32) 16416
dropout_5 (Dropout) (None, 124, 32) 0
max_pooling1d_1 (MaxPooling (None, 62, 32) 0
1D)
conv1d_3 (Conv1D) (None, 62, 16) 3088
dropout_6 (Dropout) (None, 62, 16) 0
max_pooling1d_2 (MaxPooling (None, 31, 16) 0
1D)
conv1d_4 (Conv1D) (None, 31, 8) 776
dropout_7 (Dropout) (None, 31, 8) 0
max_pooling1d_3 (MaxPooling (None, 15, 8) 0
1D)
conv1d_5 (Conv1D) (None, 15, 8) 392
dropout_8 (Dropout) (None, 15, 8) 0
max_pooling1d_4 (MaxPooling (None, 7, 8) 0
1D)
flatten (Flatten) (None, 56) 0
dense_4 (Dense) (None, 128) 7296
dropout_9 (Dropout) (None, 128) 0
dense_5 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d (Conv1D) (None, 249, 160) 5280 ['sequence_data[0][0]']
dropout_3 (Dropout) (None, 249, 160) 0 ['conv1d[0][0]']
conv1d_1 (Conv1D) (None, 249, 64) 81984 ['dropout_3[0][0]']
dropout_4 (Dropout) (None, 249, 64) 0 ['conv1d_1[0][0]']
max_pooling1d (MaxPooling1D) (None, 124, 64) 0 ['dropout_4[0][0]']
conv1d_2 (Conv1D) (None, 124, 32) 16416 ['max_pooling1d[0][0]']
dropout_5 (Dropout) (None, 124, 32) 0 ['conv1d_2[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 62, 32) 0 ['dropout_5[0][0]']
conv1d_3 (Conv1D) (None, 62, 16) 3088 ['max_pooling1d_1[0][0]']
dropout_6 (Dropout) (None, 62, 16) 0 ['conv1d_3[0][0]']
max_pooling1d_2 (MaxPooling1D) (None, 31, 16) 0 ['dropout_6[0][0]']
conv1d_4 (Conv1D) (None, 31, 8) 776 ['max_pooling1d_2[0][0]']
dropout_7 (Dropout) (None, 31, 8) 0 ['conv1d_4[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_3 (MaxPooling1D) (None, 15, 8) 0 ['dropout_7[0][0]']
dense (Dense) (None, 128) 71936 ['epigenomic_data[0][0]']
conv1d_5 (Conv1D) (None, 15, 8) 392 ['max_pooling1d_3[0][0]']
dropout (Dropout) (None, 128) 0 ['dense[0][0]']
dropout_8 (Dropout) (None, 15, 8) 0 ['conv1d_5[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dropout[0][0]']
max_pooling1d_4 (MaxPooling1D) (None, 7, 8) 0 ['dropout_8[0][0]']
dropout_1 (Dropout) (None, 64) 0 ['dense_1[0][0]']
flatten (Flatten) (None, 56) 0 ['max_pooling1d_4[0][0]']
dense_2 (Dense) (None, 64) 4160 ['dropout_1[0][0]']
dense_4 (Dense) (None, 128) 7296 ['flatten[0][0]']
dropout_2 (Dropout) (None, 64) 0 ['dense_2[0][0]']
dropout_9 (Dropout) (None, 128) 0 ['dense_4[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dropout_2[0][0]',
'dropout_9[0][0]']
dense_6 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_7 (Dense) (None, 1) 65 ['dense_6[0][0]']
==================================================================================================
Total params: 212,001
Trainable params: 212,001
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 20s 76ms/step - loss: 16.7347 - accuracy: 0.8936 - recall: 0.2456 - precision: 0.5720 - AUROC: 0.7452 - AUPRC: 0.3881 - f1_score: 0.3436 - balanced_accuracy: 0.6110 - specificity: 0.9765 - miss_rate: 0.7544 - fall_out: 0.0235 - mcc: 0.3272 - val_loss: 2.4387 - val_accuracy: 0.8997 - val_recall: 0.1317 - val_precision: 0.8873 - val_AUROC: 0.8359 - val_AUPRC: 0.5579 - val_f1_score: 0.2294 - val_balanced_accuracy: 0.5648 - val_specificity: 0.9979 - val_miss_rate: 0.8683 - val_fall_out: 0.0021 - val_mcc: 0.3194
Epoch 2/1000
198/198 [==============================] - 14s 67ms/step - loss: 2.0127 - accuracy: 0.9080 - recall: 0.3100 - precision: 0.7192 - AUROC: 0.8275 - AUPRC: 0.5275 - f1_score: 0.4332 - balanced_accuracy: 0.6473 - specificity: 0.9845 - miss_rate: 0.6900 - fall_out: 0.0155 - mcc: 0.4331 - val_loss: 2.0498 - val_accuracy: 0.9029 - val_recall: 0.1624 - val_precision: 0.8962 - val_AUROC: 0.8497 - val_AUPRC: 0.5688 - val_f1_score: 0.2749 - val_balanced_accuracy: 0.5800 - val_specificity: 0.9976 - val_miss_rate: 0.8376 - val_fall_out: 0.0024 - val_mcc: 0.3575
Epoch 3/1000
198/198 [==============================] - 14s 68ms/step - loss: 1.8098 - accuracy: 0.9100 - recall: 0.3305 - precision: 0.7269 - AUROC: 0.8414 - AUPRC: 0.5513 - f1_score: 0.4544 - balanced_accuracy: 0.6573 - specificity: 0.9841 - miss_rate: 0.6695 - fall_out: 0.0159 - mcc: 0.4512 - val_loss: 1.9909 - val_accuracy: 0.9068 - val_recall: 0.2209 - val_precision: 0.8386 - val_AUROC: 0.8506 - val_AUPRC: 0.5733 - val_f1_score: 0.3497 - val_balanced_accuracy: 0.6077 - val_specificity: 0.9946 - val_miss_rate: 0.7791 - val_fall_out: 0.0054 - val_mcc: 0.4013
Epoch 4/1000
198/198 [==============================] - 14s 68ms/step - loss: 1.6907 - accuracy: 0.9115 - recall: 0.3384 - precision: 0.7399 - AUROC: 0.8465 - AUPRC: 0.5640 - f1_score: 0.4644 - balanced_accuracy: 0.6616 - specificity: 0.9848 - miss_rate: 0.6616 - fall_out: 0.0152 - mcc: 0.4621 - val_loss: 1.8890 - val_accuracy: 0.9072 - val_recall: 0.2160 - val_precision: 0.8635 - val_AUROC: 0.8531 - val_AUPRC: 0.5795 - val_f1_score: 0.3456 - val_balanced_accuracy: 0.6058 - val_specificity: 0.9956 - val_miss_rate: 0.7840 - val_fall_out: 0.0044 - val_mcc: 0.4042
Epoch 5/1000
198/198 [==============================] - 15s 70ms/step - loss: 1.5981 - accuracy: 0.9132 - recall: 0.3520 - precision: 0.7494 - AUROC: 0.8557 - AUPRC: 0.5791 - f1_score: 0.4790 - balanced_accuracy: 0.6685 - specificity: 0.9849 - miss_rate: 0.6480 - fall_out: 0.0151 - mcc: 0.4757 - val_loss: 1.7815 - val_accuracy: 0.9065 - val_recall: 0.1993 - val_precision: 0.8910 - val_AUROC: 0.8499 - val_AUPRC: 0.5812 - val_f1_score: 0.3257 - val_balanced_accuracy: 0.5981 - val_specificity: 0.9969 - val_miss_rate: 0.8007 - val_fall_out: 0.0031 - val_mcc: 0.3956
Epoch 6/1000
198/198 [==============================] - 14s 68ms/step - loss: 1.5263 - accuracy: 0.9122 - recall: 0.3450 - precision: 0.7439 - AUROC: 0.8601 - AUPRC: 0.5813 - f1_score: 0.4714 - balanced_accuracy: 0.6649 - specificity: 0.9848 - miss_rate: 0.6550 - fall_out: 0.0152 - mcc: 0.4685 - val_loss: 1.6883 - val_accuracy: 0.9062 - val_recall: 0.1972 - val_precision: 0.8899 - val_AUROC: 0.8551 - val_AUPRC: 0.5819 - val_f1_score: 0.3229 - val_balanced_accuracy: 0.5970 - val_specificity: 0.9969 - val_miss_rate: 0.8028 - val_fall_out: 0.0031 - val_mcc: 0.3932
Epoch 7/1000
198/198 [==============================] - 14s 68ms/step - loss: 1.4682 - accuracy: 0.9134 - recall: 0.3561 - precision: 0.7485 - AUROC: 0.8631 - AUPRC: 0.5897 - f1_score: 0.4827 - balanced_accuracy: 0.6704 - specificity: 0.9847 - miss_rate: 0.6439 - fall_out: 0.0153 - mcc: 0.4784 - val_loss: 1.6691 - val_accuracy: 0.9114 - val_recall: 0.2683 - val_precision: 0.8443 - val_AUROC: 0.8556 - val_AUPRC: 0.5875 - val_f1_score: 0.4072 - val_balanced_accuracy: 0.6310 - val_specificity: 0.9937 - val_miss_rate: 0.7317 - val_fall_out: 0.0063 - val_mcc: 0.4457
Epoch 8/1000
198/198 [==============================] - 15s 71ms/step - loss: 1.4179 - accuracy: 0.9129 - recall: 0.3633 - precision: 0.7345 - AUROC: 0.8680 - AUPRC: 0.5995 - f1_score: 0.4861 - balanced_accuracy: 0.6732 - specificity: 0.9832 - miss_rate: 0.6367 - fall_out: 0.0168 - mcc: 0.4775 - val_loss: 1.5363 - val_accuracy: 0.9061 - val_recall: 0.1909 - val_precision: 0.9073 - val_AUROC: 0.8521 - val_AUPRC: 0.5806 - val_f1_score: 0.3155 - val_balanced_accuracy: 0.5942 - val_specificity: 0.9975 - val_miss_rate: 0.8091 - val_fall_out: 0.0025 - val_mcc: 0.3915
Epoch 9/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.3743 - accuracy: 0.9151 - recall: 0.3718 - precision: 0.7558 - AUROC: 0.8724 - AUPRC: 0.6087 - f1_score: 0.4984 - balanced_accuracy: 0.6782 - specificity: 0.9846 - miss_rate: 0.6282 - fall_out: 0.0154 - mcc: 0.4924 - val_loss: 1.5672 - val_accuracy: 0.9095 - val_recall: 0.2369 - val_precision: 0.8696 - val_AUROC: 0.8480 - val_AUPRC: 0.5812 - val_f1_score: 0.3724 - val_balanced_accuracy: 0.6162 - val_specificity: 0.9955 - val_miss_rate: 0.7631 - val_fall_out: 0.0045 - val_mcc: 0.4258
Epoch 10/1000
198/198 [==============================] - 14s 68ms/step - loss: 1.3372 - accuracy: 0.9144 - recall: 0.3669 - precision: 0.7506 - AUROC: 0.8722 - AUPRC: 0.6108 - f1_score: 0.4929 - balanced_accuracy: 0.6757 - specificity: 0.9844 - miss_rate: 0.6331 - fall_out: 0.0156 - mcc: 0.4869 - val_loss: 1.5152 - val_accuracy: 0.9102 - val_recall: 0.2537 - val_precision: 0.8465 - val_AUROC: 0.8537 - val_AUPRC: 0.5834 - val_f1_score: 0.3903 - val_balanced_accuracy: 0.6239 - val_specificity: 0.9941 - val_miss_rate: 0.7463 - val_fall_out: 0.0059 - val_mcc: 0.4336
Epoch 11/1000
198/198 [==============================] - 14s 69ms/step - loss: 1.3025 - accuracy: 0.9153 - recall: 0.3758 - precision: 0.7538 - AUROC: 0.8779 - AUPRC: 0.6169 - f1_score: 0.5016 - balanced_accuracy: 0.6801 - specificity: 0.9843 - miss_rate: 0.6242 - fall_out: 0.0157 - mcc: 0.4944 - val_loss: 1.5144 - val_accuracy: 0.9086 - val_recall: 0.2258 - val_precision: 0.8757 - val_AUROC: 0.8541 - val_AUPRC: 0.5872 - val_f1_score: 0.3590 - val_balanced_accuracy: 0.6108 - val_specificity: 0.9959 - val_miss_rate: 0.7742 - val_fall_out: 0.0041 - val_mcc: 0.4172
Epoch 12/1000
198/198 [==============================] - 14s 68ms/step - loss: 1.2728 - accuracy: 0.9163 - recall: 0.3817 - precision: 0.7606 - AUROC: 0.8783 - AUPRC: 0.6216 - f1_score: 0.5083 - balanced_accuracy: 0.6832 - specificity: 0.9846 - miss_rate: 0.6183 - fall_out: 0.0154 - mcc: 0.5014 - val_loss: 1.4657 - val_accuracy: 0.9108 - val_recall: 0.2662 - val_precision: 0.8341 - val_AUROC: 0.8516 - val_AUPRC: 0.5834 - val_f1_score: 0.4036 - val_balanced_accuracy: 0.6297 - val_specificity: 0.9932 - val_miss_rate: 0.7338 - val_fall_out: 0.0068 - val_mcc: 0.4404
Epoch 13/1000
198/198 [==============================] - 15s 73ms/step - loss: 1.2420 - accuracy: 0.9165 - recall: 0.3993 - precision: 0.7467 - AUROC: 0.8833 - AUPRC: 0.6305 - f1_score: 0.5204 - balanced_accuracy: 0.6910 - specificity: 0.9827 - miss_rate: 0.6007 - fall_out: 0.0173 - mcc: 0.5075 - val_loss: 1.4069 - val_accuracy: 0.9099 - val_recall: 0.2544 - val_precision: 0.8391 - val_AUROC: 0.8492 - val_AUPRC: 0.5800 - val_f1_score: 0.3904 - val_balanced_accuracy: 0.6241 - val_specificity: 0.9938 - val_miss_rate: 0.7456 - val_fall_out: 0.0062 - val_mcc: 0.4318
Epoch 14/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.2160 - accuracy: 0.9161 - recall: 0.3966 - precision: 0.7441 - AUROC: 0.8887 - AUPRC: 0.6312 - f1_score: 0.5174 - balanced_accuracy: 0.6896 - specificity: 0.9826 - miss_rate: 0.6034 - fall_out: 0.0174 - mcc: 0.5045 - val_loss: 1.3539 - val_accuracy: 0.9130 - val_recall: 0.2934 - val_precision: 0.8287 - val_AUROC: 0.8505 - val_AUPRC: 0.5841 - val_f1_score: 0.4334 - val_balanced_accuracy: 0.6428 - val_specificity: 0.9922 - val_miss_rate: 0.7066 - val_fall_out: 0.0078 - val_mcc: 0.4614
Epoch 15/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.1912 - accuracy: 0.9164 - recall: 0.4002 - precision: 0.7444 - AUROC: 0.8910 - AUPRC: 0.6387 - f1_score: 0.5206 - balanced_accuracy: 0.6913 - specificity: 0.9824 - miss_rate: 0.5998 - fall_out: 0.0176 - mcc: 0.5071 - val_loss: 1.3304 - val_accuracy: 0.9121 - val_recall: 0.2906 - val_precision: 0.8160 - val_AUROC: 0.8422 - val_AUPRC: 0.5800 - val_f1_score: 0.4286 - val_balanced_accuracy: 0.6411 - val_specificity: 0.9916 - val_miss_rate: 0.7094 - val_fall_out: 0.0084 - val_mcc: 0.4546
Epoch 16/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.1675 - accuracy: 0.9170 - recall: 0.4100 - precision: 0.7433 - AUROC: 0.8938 - AUPRC: 0.6444 - f1_score: 0.5285 - balanced_accuracy: 0.6959 - specificity: 0.9819 - miss_rate: 0.5900 - fall_out: 0.0181 - mcc: 0.5131 - val_loss: 1.2629 - val_accuracy: 0.9131 - val_recall: 0.3066 - val_precision: 0.8073 - val_AUROC: 0.8431 - val_AUPRC: 0.5795 - val_f1_score: 0.4444 - val_balanced_accuracy: 0.6486 - val_specificity: 0.9906 - val_miss_rate: 0.6934 - val_fall_out: 0.0094 - val_mcc: 0.4643
Epoch 17/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.1451 - accuracy: 0.9175 - recall: 0.4211 - precision: 0.7392 - AUROC: 0.8960 - AUPRC: 0.6503 - f1_score: 0.5366 - balanced_accuracy: 0.7011 - specificity: 0.9810 - miss_rate: 0.5789 - fall_out: 0.0190 - mcc: 0.5187 - val_loss: 1.2453 - val_accuracy: 0.9121 - val_recall: 0.3045 - val_precision: 0.7931 - val_AUROC: 0.8460 - val_AUPRC: 0.5758 - val_f1_score: 0.4401 - val_balanced_accuracy: 0.6472 - val_specificity: 0.9898 - val_miss_rate: 0.6955 - val_fall_out: 0.0102 - val_mcc: 0.4574
Epoch 18/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.1221 - accuracy: 0.9184 - recall: 0.4366 - precision: 0.7365 - AUROC: 0.9008 - AUPRC: 0.6582 - f1_score: 0.5482 - balanced_accuracy: 0.7083 - specificity: 0.9800 - miss_rate: 0.5634 - fall_out: 0.0200 - mcc: 0.5275 - val_loss: 1.2082 - val_accuracy: 0.9125 - val_recall: 0.3101 - val_precision: 0.7904 - val_AUROC: 0.8386 - val_AUPRC: 0.5689 - val_f1_score: 0.4454 - val_balanced_accuracy: 0.6498 - val_specificity: 0.9895 - val_miss_rate: 0.6899 - val_fall_out: 0.0105 - val_mcc: 0.4607
Epoch 19/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.1015 - accuracy: 0.9192 - recall: 0.4568 - precision: 0.7298 - AUROC: 0.9029 - AUPRC: 0.6658 - f1_score: 0.5619 - balanced_accuracy: 0.7176 - specificity: 0.9784 - miss_rate: 0.5432 - fall_out: 0.0216 - mcc: 0.5374 - val_loss: 1.1651 - val_accuracy: 0.9120 - val_recall: 0.3380 - val_precision: 0.7473 - val_AUROC: 0.8412 - val_AUPRC: 0.5726 - val_f1_score: 0.4655 - val_balanced_accuracy: 0.6617 - val_specificity: 0.9854 - val_miss_rate: 0.6620 - val_fall_out: 0.0146 - val_mcc: 0.4648
Epoch 20/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.0812 - accuracy: 0.9194 - recall: 0.4620 - precision: 0.7274 - AUROC: 0.9056 - AUPRC: 0.6721 - f1_score: 0.5651 - balanced_accuracy: 0.7199 - specificity: 0.9779 - miss_rate: 0.5380 - fall_out: 0.0221 - mcc: 0.5395 - val_loss: 1.1795 - val_accuracy: 0.9125 - val_recall: 0.3338 - val_precision: 0.7591 - val_AUROC: 0.8431 - val_AUPRC: 0.5724 - val_f1_score: 0.4637 - val_balanced_accuracy: 0.6601 - val_specificity: 0.9865 - val_miss_rate: 0.6662 - val_fall_out: 0.0135 - val_mcc: 0.4665
Epoch 21/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.0642 - accuracy: 0.9210 - recall: 0.4815 - precision: 0.7303 - AUROC: 0.9062 - AUPRC: 0.6724 - f1_score: 0.5804 - balanced_accuracy: 0.7294 - specificity: 0.9773 - miss_rate: 0.5185 - fall_out: 0.0227 - mcc: 0.5531 - val_loss: 1.1152 - val_accuracy: 0.9105 - val_recall: 0.3554 - val_precision: 0.7103 - val_AUROC: 0.8375 - val_AUPRC: 0.5693 - val_f1_score: 0.4738 - val_balanced_accuracy: 0.6684 - val_specificity: 0.9815 - val_miss_rate: 0.6446 - val_fall_out: 0.0185 - val_mcc: 0.4617
Epoch 22/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.0438 - accuracy: 0.9214 - recall: 0.5044 - precision: 0.7190 - AUROC: 0.9090 - AUPRC: 0.6813 - f1_score: 0.5928 - balanced_accuracy: 0.7396 - specificity: 0.9748 - miss_rate: 0.4956 - fall_out: 0.0252 - mcc: 0.5614 - val_loss: 1.1104 - val_accuracy: 0.9121 - val_recall: 0.3373 - val_precision: 0.7492 - val_AUROC: 0.8389 - val_AUPRC: 0.5677 - val_f1_score: 0.4652 - val_balanced_accuracy: 0.6614 - val_specificity: 0.9856 - val_miss_rate: 0.6627 - val_fall_out: 0.0144 - val_mcc: 0.4651
Epoch 23/1000
198/198 [==============================] - 14s 68ms/step - loss: 1.0255 - accuracy: 0.9232 - recall: 0.5287 - precision: 0.7196 - AUROC: 0.9117 - AUPRC: 0.6879 - f1_score: 0.6096 - balanced_accuracy: 0.7512 - specificity: 0.9736 - miss_rate: 0.4713 - fall_out: 0.0264 - mcc: 0.5764 - val_loss: 1.0948 - val_accuracy: 0.9099 - val_recall: 0.3303 - val_precision: 0.7259 - val_AUROC: 0.8347 - val_AUPRC: 0.5566 - val_f1_score: 0.4540 - val_balanced_accuracy: 0.6572 - val_specificity: 0.9840 - val_miss_rate: 0.6697 - val_fall_out: 0.0160 - val_mcc: 0.4506
Epoch 24/1000
198/198 [==============================] - 14s 67ms/step - loss: 1.0084 - accuracy: 0.9216 - recall: 0.5232 - precision: 0.7097 - AUROC: 0.9148 - AUPRC: 0.6889 - f1_score: 0.6023 - balanced_accuracy: 0.7479 - specificity: 0.9726 - miss_rate: 0.4768 - fall_out: 0.0274 - mcc: 0.5680 - val_loss: 1.0773 - val_accuracy: 0.9116 - val_recall: 0.3470 - val_precision: 0.7324 - val_AUROC: 0.8341 - val_AUPRC: 0.5639 - val_f1_score: 0.4709 - val_balanced_accuracy: 0.6654 - val_specificity: 0.9838 - val_miss_rate: 0.6530 - val_fall_out: 0.0162 - val_mcc: 0.4652
Epoch 25/1000
198/198 [==============================] - 14s 67ms/step - loss: 0.9885 - accuracy: 0.9244 - recall: 0.5402 - precision: 0.7229 - AUROC: 0.9194 - AUPRC: 0.7013 - f1_score: 0.6184 - balanced_accuracy: 0.7569 - specificity: 0.9735 - miss_rate: 0.4598 - fall_out: 0.0265 - mcc: 0.5849 - val_loss: 1.0613 - val_accuracy: 0.9089 - val_recall: 0.3742 - val_precision: 0.6780 - val_AUROC: 0.8272 - val_AUPRC: 0.5589 - val_f1_score: 0.4823 - val_balanced_accuracy: 0.6757 - val_specificity: 0.9773 - val_miss_rate: 0.6258 - val_fall_out: 0.0227 - val_mcc: 0.4601
Epoch 26/1000
198/198 [==============================] - 14s 67ms/step - loss: 0.9724 - accuracy: 0.9256 - recall: 0.5637 - precision: 0.7197 - AUROC: 0.9193 - AUPRC: 0.7077 - f1_score: 0.6322 - balanced_accuracy: 0.7678 - specificity: 0.9719 - miss_rate: 0.4363 - fall_out: 0.0281 - mcc: 0.5970 - val_loss: 1.0528 - val_accuracy: 0.9095 - val_recall: 0.3923 - val_precision: 0.6734 - val_AUROC: 0.8351 - val_AUPRC: 0.5659 - val_f1_score: 0.4958 - val_balanced_accuracy: 0.6840 - val_specificity: 0.9757 - val_miss_rate: 0.6077 - val_fall_out: 0.0243 - val_mcc: 0.4698
Epoch 27/1000
198/198 [==============================] - 14s 67ms/step - loss: 0.9535 - accuracy: 0.9275 - recall: 0.5709 - precision: 0.7312 - AUROC: 0.9251 - AUPRC: 0.7187 - f1_score: 0.6412 - balanced_accuracy: 0.7720 - specificity: 0.9732 - miss_rate: 0.4291 - fall_out: 0.0268 - mcc: 0.6073 - val_loss: 1.0394 - val_accuracy: 0.9082 - val_recall: 0.4181 - val_precision: 0.6472 - val_AUROC: 0.8321 - val_AUPRC: 0.5571 - val_f1_score: 0.5080 - val_balanced_accuracy: 0.6945 - val_specificity: 0.9709 - val_miss_rate: 0.5819 - val_fall_out: 0.0291 - val_mcc: 0.4734
Epoch 28/1000
198/198 [==============================] - 14s 67ms/step - loss: 0.9371 - accuracy: 0.9286 - recall: 0.5914 - precision: 0.7281 - AUROC: 0.9279 - AUPRC: 0.7248 - f1_score: 0.6527 - balanced_accuracy: 0.7816 - specificity: 0.9718 - miss_rate: 0.4086 - fall_out: 0.0282 - mcc: 0.6175 - val_loss: 1.0030 - val_accuracy: 0.9101 - val_recall: 0.3749 - val_precision: 0.6906 - val_AUROC: 0.8276 - val_AUPRC: 0.5584 - val_f1_score: 0.4860 - val_balanced_accuracy: 0.6767 - val_specificity: 0.9785 - val_miss_rate: 0.6251 - val_fall_out: 0.0215 - val_mcc: 0.4663
Epoch 29/1000
198/198 [==============================] - 14s 68ms/step - loss: 0.9232 - accuracy: 0.9293 - recall: 0.5927 - precision: 0.7331 - AUROC: 0.9289 - AUPRC: 0.7292 - f1_score: 0.6554 - balanced_accuracy: 0.7825 - specificity: 0.9724 - miss_rate: 0.4073 - fall_out: 0.0276 - mcc: 0.6209 - val_loss: 0.9975 - val_accuracy: 0.9106 - val_recall: 0.3603 - val_precision: 0.7082 - val_AUROC: 0.8314 - val_AUPRC: 0.5581 - val_f1_score: 0.4776 - val_balanced_accuracy: 0.6706 - val_specificity: 0.9810 - val_miss_rate: 0.6397 - val_fall_out: 0.0190 - val_mcc: 0.4642
Epoch 30/1000
198/198 [==============================] - 14s 68ms/step - loss: 0.9078 - accuracy: 0.9306 - recall: 0.5871 - precision: 0.7466 - AUROC: 0.9303 - AUPRC: 0.7362 - f1_score: 0.6573 - balanced_accuracy: 0.7808 - specificity: 0.9745 - miss_rate: 0.4129 - fall_out: 0.0255 - mcc: 0.6248 - val_loss: 0.9852 - val_accuracy: 0.9092 - val_recall: 0.3526 - val_precision: 0.6970 - val_AUROC: 0.8344 - val_AUPRC: 0.5568 - val_f1_score: 0.4683 - val_balanced_accuracy: 0.6665 - val_specificity: 0.9804 - val_miss_rate: 0.6474 - val_fall_out: 0.0196 - val_mcc: 0.4541
Epoch 31/1000
198/198 [==============================] - 14s 67ms/step - loss: 0.8923 - accuracy: 0.9316 - recall: 0.5993 - precision: 0.7477 - AUROC: 0.9342 - AUPRC: 0.7429 - f1_score: 0.6653 - balanced_accuracy: 0.7867 - specificity: 0.9741 - miss_rate: 0.4007 - fall_out: 0.0259 - mcc: 0.6325 - val_loss: 0.9850 - val_accuracy: 0.9099 - val_recall: 0.3742 - val_precision: 0.6885 - val_AUROC: 0.8224 - val_AUPRC: 0.5523 - val_f1_score: 0.4849 - val_balanced_accuracy: 0.6763 - val_specificity: 0.9783 - val_miss_rate: 0.6258 - val_fall_out: 0.0217 - val_mcc: 0.4648
Epoch 32/1000
198/198 [==============================] - 14s 68ms/step - loss: 0.8794 - accuracy: 0.9318 - recall: 0.5984 - precision: 0.7497 - AUROC: 0.9346 - AUPRC: 0.7442 - f1_score: 0.6656 - balanced_accuracy: 0.7864 - specificity: 0.9744 - miss_rate: 0.4016 - fall_out: 0.0256 - mcc: 0.6331 - val_loss: 1.0116 - val_accuracy: 0.9107 - val_recall: 0.3449 - val_precision: 0.7226 - val_AUROC: 0.8164 - val_AUPRC: 0.5468 - val_f1_score: 0.4670 - val_balanced_accuracy: 0.6640 - val_specificity: 0.9831 - val_miss_rate: 0.6551 - val_fall_out: 0.0169 - val_mcc: 0.4597
Epoch 33/1000
198/198 [==============================] - 14s 67ms/step - loss: 0.8658 - accuracy: 0.9346 - recall: 0.6209 - precision: 0.7585 - AUROC: 0.9374 - AUPRC: 0.7518 - f1_score: 0.6828 - balanced_accuracy: 0.7978 - specificity: 0.9747 - miss_rate: 0.3791 - fall_out: 0.0253 - mcc: 0.6508 - val_loss: 0.9847 - val_accuracy: 0.9111 - val_recall: 0.3491 - val_precision: 0.7240 - val_AUROC: 0.8286 - val_AUPRC: 0.5508 - val_f1_score: 0.4711 - val_balanced_accuracy: 0.6661 - val_specificity: 0.9830 - val_miss_rate: 0.6509 - val_fall_out: 0.0170 - val_mcc: 0.4632
Epoch 34/1000
198/198 [==============================] - 14s 67ms/step - loss: 0.8502 - accuracy: 0.9345 - recall: 0.6250 - precision: 0.7554 - AUROC: 0.9393 - AUPRC: 0.7610 - f1_score: 0.6841 - balanced_accuracy: 0.7996 - specificity: 0.9741 - miss_rate: 0.3750 - fall_out: 0.0259 - mcc: 0.6515 - val_loss: 0.9378 - val_accuracy: 0.9090 - val_recall: 0.3784 - val_precision: 0.6762 - val_AUROC: 0.8244 - val_AUPRC: 0.5513 - val_f1_score: 0.4853 - val_balanced_accuracy: 0.6776 - val_specificity: 0.9768 - val_miss_rate: 0.6216 - val_fall_out: 0.0232 - val_mcc: 0.4620
Epoch 35/1000
198/198 [==============================] - 14s 67ms/step - loss: 0.8384 - accuracy: 0.9360 - recall: 0.6296 - precision: 0.7643 - AUROC: 0.9408 - AUPRC: 0.7629 - f1_score: 0.6904 - balanced_accuracy: 0.8024 - specificity: 0.9752 - miss_rate: 0.3704 - fall_out: 0.0248 - mcc: 0.6589 - val_loss: 0.9935 - val_accuracy: 0.9091 - val_recall: 0.3443 - val_precision: 0.7017 - val_AUROC: 0.8143 - val_AUPRC: 0.5424 - val_f1_score: 0.4619 - val_balanced_accuracy: 0.6628 - val_specificity: 0.9813 - val_miss_rate: 0.6557 - val_fall_out: 0.0187 - val_mcc: 0.4503
Epoch 36/1000
198/198 [==============================] - 15s 69ms/step - loss: 0.8266 - accuracy: 0.9371 - recall: 0.6409 - precision: 0.7659 - AUROC: 0.9420 - AUPRC: 0.7700 - f1_score: 0.6978 - balanced_accuracy: 0.8079 - specificity: 0.9749 - miss_rate: 0.3591 - fall_out: 0.0251 - mcc: 0.6663 - val_loss: 0.9914 - val_accuracy: 0.9099 - val_recall: 0.3429 - val_precision: 0.7130 - val_AUROC: 0.8157 - val_AUPRC: 0.5399 - val_f1_score: 0.4631 - val_balanced_accuracy: 0.6626 - val_specificity: 0.9824 - val_miss_rate: 0.6571 - val_fall_out: 0.0176 - val_mcc: 0.4542
198/198 [==============================] - 8s 39ms/step - loss: 0.8403 - accuracy: 0.9462 - recall: 0.6007 - precision: 0.8894 - AUROC: 0.9699 - AUPRC: 0.8628 - f1_score: 0.7170 - balanced_accuracy: 0.7956 - specificity: 0.9904 - miss_rate: 0.3993 - fall_out: 0.0096 - mcc: 0.7048
50/50 [==============================] - 6s 118ms/step - loss: 0.9914 - accuracy: 0.9099 - recall: 0.3429 - precision: 0.7130 - AUROC: 0.8157 - AUPRC: 0.5399 - f1_score: 0.4631 - balanced_accuracy: 0.6626 - specificity: 0.9824 - miss_rate: 0.6571 - fall_out: 0.0176 - mcc: 0.4542
INFO:tensorflow:Reloading Oracle from existing project MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_9\oracle.json
Model: "MLP_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
dense (Dense) (None, 8) 4504
dropout (Dropout) (None, 8) 0
dense_1 (Dense) (None, 1) 9
=================================================================
Total params: 4,513
Trainable params: 4,513
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from MLP_hypermodel_enhancers\MLP_hypermodel_enhancers_9\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'depth': 3, 'units_0': 136, 'dropout_0': 0.3, 'units_1': 8, 'dropout_1': 0.3, 'units_2': 8, 'dropout_2': 0.3}
INFO:tensorflow:Reloading Oracle from existing project CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_9\oracle.json
Model: "CNN_Hypermodel"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv1d (Conv1D) (None, 256, 8) 200
dropout_1 (Dropout) (None, 256, 8) 0
max_pooling1d (MaxPooling1D (None, 128, 8) 0
)
flatten (Flatten) (None, 1024) 0
dense_2 (Dense) (None, 16) 16400
dropout_2 (Dropout) (None, 16) 0
dense_3 (Dense) (None, 1) 17
=================================================================
Total params: 16,617
Trainable params: 16,617
Non-trainable params: 0
_________________________________________________________________
INFO:tensorflow:Reloading Tuner from CNN_hypermodel_enhancers\CNN_hypermodel_enhancers_9\tuner0.json
INFO:tensorflow:Oracle triggered exit
-- Best set of hyperparameters found:
{'conv_depth': 3, 'filters_0': 96, 'kernel_size_0': 6, 'conv_dropout_0': 0.1, 'dense_depth': 1, 'units_0': 32, 'dense_dropout_0': 0.4, 'filters_1': 96, 'kernel_size_1': 12, 'conv_dropout_1': 0.2, 'filters_2': 88, 'kernel_size_2': 16, 'conv_dropout_2': 0.2, 'filters_3': 104, 'kernel_size_3': 8, 'conv_dropout_3': 0.0, 'units_1': 32, 'dense_dropout_1': 0.3, 'units_2': 96, 'dense_dropout_2': 0.4, 'units_3': 48, 'dense_dropout_3': 0.4, 'units_4': 48, 'dense_dropout_4': 0.4}
- Removing uncorrelated/correlated features: ['SNRNP70', 'ZNF282']
- Building Optimized MLP:
Model: "model"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 561)] 0
)
dense_4 (Dense) (None, 136) 76432
dropout_3 (Dropout) (None, 136) 0
dense_5 (Dense) (None, 8) 1096
dropout_4 (Dropout) (None, 8) 0
dense_6 (Dense) (None, 8) 72
dropout_5 (Dropout) (None, 8) 0
dense_7 (Dense) (None, 1) 9
=================================================================
Total params: 77,609
Trainable params: 77,609
Non-trainable params: 0
_________________________________________________________________
- Building Optimized CNN:
Model: "model_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_1 (Conv1D) (None, 256, 96) 2400
dropout_6 (Dropout) (None, 256, 96) 0
max_pooling1d_1 (MaxPooling (None, 128, 96) 0
1D)
conv1d_2 (Conv1D) (None, 128, 96) 110688
dropout_7 (Dropout) (None, 128, 96) 0
max_pooling1d_2 (MaxPooling (None, 64, 96) 0
1D)
conv1d_3 (Conv1D) (None, 64, 88) 135256
dropout_8 (Dropout) (None, 64, 88) 0
max_pooling1d_3 (MaxPooling (None, 32, 88) 0
1D)
flatten_1 (Flatten) (None, 2816) 0
dense_8 (Dense) (None, 32) 90144
dropout_9 (Dropout) (None, 32) 0
dense_9 (Dense) (None, 1) 33
=================================================================
Total params: 338,521
Trainable params: 338,521
Non-trainable params: 0
_________________________________________________________________
- Building MMNN:
Model: "model_2"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0 []
conv1d_1 (Conv1D) (None, 256, 96) 2400 ['sequence_data[0][0]']
dropout_6 (Dropout) (None, 256, 96) 0 ['conv1d_1[0][0]']
max_pooling1d_1 (MaxPooling1D) (None, 128, 96) 0 ['dropout_6[0][0]']
conv1d_2 (Conv1D) (None, 128, 96) 110688 ['max_pooling1d_1[0][0]']
dropout_7 (Dropout) (None, 128, 96) 0 ['conv1d_2[0][0]']
epigenomic_data (InputLayer) [(None, 561)] 0 []
max_pooling1d_2 (MaxPooling1D) (None, 64, 96) 0 ['dropout_7[0][0]']
dense_4 (Dense) (None, 136) 76432 ['epigenomic_data[0][0]']
conv1d_3 (Conv1D) (None, 64, 88) 135256 ['max_pooling1d_2[0][0]']
dropout_3 (Dropout) (None, 136) 0 ['dense_4[0][0]']
dropout_8 (Dropout) (None, 64, 88) 0 ['conv1d_3[0][0]']
dense_5 (Dense) (None, 8) 1096 ['dropout_3[0][0]']
max_pooling1d_3 (MaxPooling1D) (None, 32, 88) 0 ['dropout_8[0][0]']
dropout_4 (Dropout) (None, 8) 0 ['dense_5[0][0]']
flatten_1 (Flatten) (None, 2816) 0 ['max_pooling1d_3[0][0]']
dense_6 (Dense) (None, 8) 72 ['dropout_4[0][0]']
dense_8 (Dense) (None, 32) 90144 ['flatten_1[0][0]']
dropout_5 (Dropout) (None, 8) 0 ['dense_6[0][0]']
dropout_9 (Dropout) (None, 32) 0 ['dense_8[0][0]']
concatenate (Concatenate) (None, 40) 0 ['dropout_5[0][0]',
'dropout_9[0][0]']
dense_10 (Dense) (None, 64) 2624 ['concatenate[0][0]']
dense_11 (Dense) (None, 1) 65 ['dense_10[0][0]']
==================================================================================================
Total params: 418,777
Trainable params: 418,777
Non-trainable params: 0
__________________________________________________________________________________________________
- Training MMNN model:
Epoch 1/1000
198/198 [==============================] - 17s 62ms/step - loss: 0.3196 - accuracy: 0.8865 - recall: 0.0000e+00 - precision: 0.0000e+00 - AUROC: 0.7440 - AUPRC: 0.2909 - f1_score: 0.0000e+00 - balanced_accuracy: 0.5000 - specificity: 1.0000 - miss_rate: 1.0000 - fall_out: 4.4557e-05 - mcc: -0.0022 - val_loss: 0.2809 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8283 - val_AUPRC: 0.4473 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 2/1000
198/198 [==============================] - 12s 57ms/step - loss: 0.2819 - accuracy: 0.8866 - recall: 0.0000e+00 - precision: 0.0000e+00 - AUROC: 0.8183 - AUPRC: 0.4240 - f1_score: 0.0000e+00 - balanced_accuracy: 0.5000 - specificity: 1.0000 - miss_rate: 1.0000 - fall_out: 0.0000e+00 - mcc: 0.0000e+00 - val_loss: 0.2951 - val_accuracy: 0.8866 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00 - val_AUROC: 0.8390 - val_AUPRC: 0.4947 - val_f1_score: 0.0000e+00 - val_balanced_accuracy: 0.5000 - val_specificity: 1.0000 - val_miss_rate: 1.0000 - val_fall_out: 0.0000e+00 - val_mcc: 0.0000e+00
Epoch 3/1000
198/198 [==============================] - 11s 55ms/step - loss: 0.2687 - accuracy: 0.8901 - recall: 0.0617 - precision: 0.6705 - AUROC: 0.8342 - AUPRC: 0.4581 - f1_score: 0.1129 - balanced_accuracy: 0.5289 - specificity: 0.9961 - miss_rate: 0.9383 - fall_out: 0.0039 - mcc: 0.1803 - val_loss: 0.2940 - val_accuracy: 0.8959 - val_recall: 0.5157 - val_precision: 0.5429 - val_AUROC: 0.8437 - val_AUPRC: 0.5009 - val_f1_score: 0.5289 - val_balanced_accuracy: 0.7301 - val_specificity: 0.9445 - val_miss_rate: 0.4843 - val_fall_out: 0.0555 - val_mcc: 0.4707
198/198 [==============================] - 8s 39ms/step - loss: 0.2849 - accuracy: 0.9021 - recall: 0.5686 - precision: 0.5685 - AUROC: 0.8695 - AUPRC: 0.5260 - f1_score: 0.5686 - balanced_accuracy: 0.7567 - specificity: 0.9448 - miss_rate: 0.4314 - fall_out: 0.0552 - mcc: 0.5134
50/50 [==============================] - 6s 112ms/step - loss: 0.2940 - accuracy: 0.8959 - recall: 0.5157 - precision: 0.5429 - AUROC: 0.8437 - AUPRC: 0.5009 - f1_score: 0.5289 - balanced_accuracy: 0.7301 - specificity: 0.9445 - miss_rate: 0.4843 - fall_out: 0.0555 - mcc: 0.4707
MMNN_metrics_estimate = model_metrics_holdout_estimate_evaluation(MMNN_metrics_enhancers["tuning_False"], number_of_splits)
print("-- ENHANCERS -- with FIXED MLP and CNN")
print(f"MMNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {MMNN_metrics_estimate['accuracy_train']} -- test - {MMNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {MMNN_metrics_estimate['AUROC_train']} -- test - {MMNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {MMNN_metrics_estimate['AUPRC_train']} -- test - {MMNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("MMNN - Train history:")
plot_train_history(MMNN_history_enhancers["tuning_False"])
print("-"*100)
MMNN_metrics_estimate = model_metrics_holdout_estimate_evaluation(MMNN_metrics_enhancers["tuning_True"], number_of_splits)
print("-- ENHANCERS -- with bayesian optimized hyperparameters MLP and CNN")
print(f"MMNN Metrics - {number_of_splits}-holdouts estimate:")
print(f"Accuracy : train - {MMNN_metrics_estimate['accuracy_train']} -- test - {MMNN_metrics_estimate['accuracy_test']}")
print(f"AUROC : train - {MMNN_metrics_estimate['AUROC_train']} -- test - {MMNN_metrics_estimate['AUROC_test']}")
print(f"AUPRC : train - {MMNN_metrics_estimate['AUPRC_train']} -- test - {MMNN_metrics_estimate['AUPRC_test']}")
print("-"*80)
print("MMNN - Train history:")
plot_train_history(MMNN_history_enhancers["tuning_True"])
-- ENHANCERS -- with FIXED MLP and CNN MMNN Metrics - 10-holdouts estimate: Accuracy : train - 0.9213261425495147 -- test - 0.9121987879276275 AUROC : train - 0.9278244793415069 -- test - 0.8479498088359833 AUPRC : train - 0.7284751772880554 -- test - 0.5957026779651642 -------------------------------------------------------------------------------- MMNN - Train history:
---------------------------------------------------------------------------------------------------- -- ENHANCERS -- with bayesian optimized hyperparameters MLP and CNN MMNN Metrics - 10-holdouts estimate: Accuracy : train - 0.9104132056236267 -- test - 0.9092122912406921 AUROC : train - 0.8816489458084107 -- test - 0.8556916177272796 AUPRC : train - 0.6161971867084504 -- test - 0.5873279452323914 -------------------------------------------------------------------------------- MMNN - Train history:
MMNN_metrics_enhancers
{'tuning_True': [{'train_evaluation': {'loss': 0.23686882853507996,
'accuracy': 0.9082720875740051,
'recall': 0.20898641645908356,
'precision': 0.921658992767334,
'AUROC': 0.8809362053871155,
'AUPRC': 0.6303306818008423,
'f1_score': 0.3407154977321625,
'balanced_accuracy': 0.6033570170402527,
'specificity': 0.9977275729179382,
'miss_rate': 0.7910135984420776,
'fall_out': 0.002272423356771469,
'mcc': 0.41410788893699646},
'test_evaluation': {'loss': 0.24579110741615295,
'accuracy': 0.9074820280075073,
'recall': 0.20209059119224548,
'precision': 0.9177215099334717,
'AUROC': 0.8608652949333191,
'AUPRC': 0.6082503795623779,
'f1_score': 0.3312392830848694,
'balanced_accuracy': 0.5998868346214294,
'specificity': 0.9976831078529358,
'miss_rate': 0.7979093790054321,
'fall_out': 0.002316877478733659,
'mcc': 0.4059576392173767}},
{'train_evaluation': {'loss': 0.22841493785381317,
'accuracy': 0.9109781384468079,
'recall': 0.23963776230812073,
'precision': 0.9070534110069275,
'AUROC': 0.8945721387863159,
'AUPRC': 0.65479576587677,
'f1_score': 0.37911558151245117,
'balanced_accuracy': 0.6182482242584229,
'specificity': 0.9968587160110474,
'miss_rate': 0.7603622674942017,
'fall_out': 0.00314129120670259,
'mcc': 0.4398752450942993},
'test_evaluation': {'loss': 0.24346749484539032,
'accuracy': 0.910879373550415,
'recall': 0.23205575346946716,
'precision': 0.9275766015052795,
'AUROC': 0.8645578622817993,
'AUPRC': 0.6120344996452332,
'f1_score': 0.37123745679855347,
'balanced_accuracy': 0.6148694157600403,
'specificity': 0.9976831078529358,
'miss_rate': 0.7679442763328552,
'fall_out': 0.002316877478733659,
'mcc': 0.43876412510871887}},
{'train_evaluation': {'loss': 0.24366118013858795,
'accuracy': 0.9126965403556824,
'recall': 0.27255311608314514,
'precision': 0.8655973672866821,
'AUROC': 0.8756171464920044,
'AUPRC': 0.6167053580284119,
'f1_score': 0.41456952691078186,
'balanced_accuracy': 0.6335697174072266,
'specificity': 0.9945862889289856,
'miss_rate': 0.7274468541145325,
'fall_out': 0.005413714796304703,
'mcc': 0.45648542046546936},
'test_evaluation': {'loss': 0.24362611770629883,
'accuracy': 0.9141976833343506,
'recall': 0.2864111363887787,
'precision': 0.8689217567443848,
'AUROC': 0.8707704544067383,
'AUPRC': 0.6272091269493103,
'f1_score': 0.4308176040649414,
'balanced_accuracy': 0.6404431462287903,
'specificity': 0.9944751262664795,
'miss_rate': 0.7135888338088989,
'fall_out': 0.005524862091988325,
'mcc': 0.4695329964160919}},
{'train_evaluation': {'loss': 0.22734124958515167,
'accuracy': 0.9117879271507263,
'recall': 0.2443399578332901,
'precision': 0.9169934391975403,
'AUROC': 0.8927640318870544,
'AUPRC': 0.6495679616928101,
'f1_score': 0.3858635723590851,
'balanced_accuracy': 0.6207553148269653,
'specificity': 0.997170627117157,
'miss_rate': 0.7556600570678711,
'fall_out': 0.002829390112310648,
'mcc': 0.44734808802604675},
'test_evaluation': {'loss': 0.24319496750831604,
'accuracy': 0.9106423258781433,
'recall': 0.24320557713508606,
'precision': 0.8857868313789368,
'AUROC': 0.8637168407440186,
'AUPRC': 0.60009765625,
'f1_score': 0.3816293179988861,
'balanced_accuracy': 0.6195977926254272,
'specificity': 0.9959900379180908,
'miss_rate': 0.7567944526672363,
'fall_out': 0.004009980242699385,
'mcc': 0.4366845190525055}},
{'train_evaluation': {'loss': 0.22470732033252716,
'accuracy': 0.9153037667274475,
'recall': 0.31661441922187805,
'precision': 0.8331805467605591,
'AUROC': 0.8888852596282959,
'AUPRC': 0.6348809003829956,
'f1_score': 0.4588591754436493,
'balanced_accuracy': 0.6542524695396423,
'specificity': 0.991890549659729,
'miss_rate': 0.6833855509757996,
'fall_out': 0.008109432645142078,
'mcc': 0.4817180335521698},
'test_evaluation': {'loss': 0.2503856122493744,
'accuracy': 0.9143556952476501,
'recall': 0.3073170781135559,
'precision': 0.8305084705352783,
'AUROC': 0.844851016998291,
'AUPRC': 0.585593581199646,
'f1_score': 0.44862666726112366,
'balanced_accuracy': 0.6496485471725464,
'specificity': 0.9919800162315369,
'miss_rate': 0.6926829218864441,
'fall_out': 0.00801996048539877,
'mcc': 0.4733229875564575}},
{'train_evaluation': {'loss': 0.26149827241897583,
'accuracy': 0.9059018492698669,
'recall': 0.18408219516277313,
'precision': 0.9304577708244324,
'AUROC': 0.868215799331665,
'AUPRC': 0.6013355255126953,
'f1_score': 0.3073568046092987,
'balanced_accuracy': 0.5911610722541809,
'specificity': 0.9982399940490723,
'miss_rate': 0.8159177899360657,
'fall_out': 0.001760014216415584,
'mcc': 0.3903641104698181},
'test_evaluation': {'loss': 0.26979291439056396,
'accuracy': 0.9045587182044983,
'recall': 0.17282229661941528,
'precision': 0.921933114528656,
'AUROC': 0.8434581160545349,
'AUPRC': 0.5716344118118286,
'f1_score': 0.2910798192024231,
'balanced_accuracy': 0.5854754447937012,
'specificity': 0.9981286525726318,
'miss_rate': 0.8271777033805847,
'fall_out': 0.0018713241443037987,
'mcc': 0.3757992684841156}},
{'train_evaluation': {'loss': 0.25664278864860535,
'accuracy': 0.9145927429199219,
'recall': 0.3009404242038727,
'precision': 0.8478900790214539,
'AUROC': 0.8753827214241028,
'AUPRC': 0.6195829510688782,
'f1_score': 0.44421592354774475,
'balanced_accuracy': 0.6470170021057129,
'specificity': 0.9930936098098755,
'miss_rate': 0.6990595459938049,
'fall_out': 0.006906385067850351,
'mcc': 0.47436121106147766},
'test_evaluation': {'loss': 0.2623143792152405,
'accuracy': 0.9132496118545532,
'recall': 0.3038327395915985,
'precision': 0.8149532675743103,
'AUROC': 0.8591158390045166,
'AUPRC': 0.602439820766449,
'f1_score': 0.4426395893096924,
'balanced_accuracy': 0.6475054025650024,
'specificity': 0.991178035736084,
'miss_rate': 0.6961672306060791,
'fall_out': 0.008821956813335419,
'mcc': 0.46487370133399963}},
{'train_evaluation': {'loss': 0.24408887326717377,
'accuracy': 0.9062376618385315,
'recall': 0.25357019901275635,
'precision': 0.7595200538635254,
'AUROC': 0.8672508597373962,
'AUPRC': 0.5737784504890442,
'f1_score': 0.38020628690719604,
'balanced_accuracy': 0.6216498613357544,
'specificity': 0.9897295236587524,
'miss_rate': 0.7464298009872437,
'fall_out': 0.010270463302731514,
'mcc': 0.4042071998119354},
'test_evaluation': {'loss': 0.25224247574806213,
'accuracy': 0.9047167301177979,
'recall': 0.2418118417263031,
'precision': 0.7462365627288818,
'AUROC': 0.8477284908294678,
'AUPRC': 0.553510844707489,
'f1_score': 0.3652631640434265,
'balanced_accuracy': 0.6156483888626099,
'specificity': 0.989484965801239,
'miss_rate': 0.7581881284713745,
'fall_out': 0.010515059344470501,
'mcc': 0.3898223340511322}},
{'train_evaluation': {'loss': 0.22102424502372742,
'accuracy': 0.9162321090698242,
'recall': 0.32479971647262573,
'precision': 0.836698055267334,
'AUROC': 0.9033868312835693,
'AUPRC': 0.6550315618515015,
'f1_score': 0.46794629096984863,
'balanced_accuracy': 0.658345103263855,
'specificity': 0.991890549659729,
'miss_rate': 0.6752002835273743,
'fall_out': 0.008109432645142078,
'mcc': 0.4894953668117523},
'test_evaluation': {'loss': 0.24087484180927277,
'accuracy': 0.9161728620529175,
'recall': 0.3296167254447937,
'precision': 0.8269230723381042,
'AUROC': 0.8581576347351074,
'AUPRC': 0.6116220951080322,
'f1_score': 0.4713502824306488,
'balanced_accuracy': 0.6603974103927612,
'specificity': 0.991178035736084,
'miss_rate': 0.6703832745552063,
'fall_out': 0.008821956813335419,
'mcc': 0.4896288812160492}},
{'train_evaluation': {'loss': 0.28490525484085083,
'accuracy': 0.9021292328834534,
'recall': 0.5686172246932983,
'precision': 0.5685182213783264,
'AUROC': 0.8694784641265869,
'AUPRC': 0.5259627103805542,
'f1_score': 0.56856769323349,
'balanced_accuracy': 0.7567053437232971,
'specificity': 0.9447934627532959,
'miss_rate': 0.43138280510902405,
'fall_out': 0.05520652234554291,
'mcc': 0.5133716464042664},
'test_evaluation': {'loss': 0.29400038719177246,
'accuracy': 0.8958678841590881,
'recall': 0.5156794190406799,
'precision': 0.5429200530052185,
'AUROC': 0.8436946272850037,
'AUPRC': 0.5008870363235474,
'f1_score': 0.5289492607116699,
'balanced_accuracy': 0.7300817370414734,
'specificity': 0.9444840550422668,
'miss_rate': 0.4843205511569977,
'fall_out': 0.05551595240831375,
'mcc': 0.4706536829471588}}],
'tuning_False': [{'train_evaluation': {'loss': 0.936881422996521,
'accuracy': 0.9281030297279358,
'recall': 0.45280390977859497,
'precision': 0.8392511010169983,
'AUROC': 0.9538806080818176,
'AUPRC': 0.7949593663215637,
'f1_score': 0.5882353186607361,
'balanced_accuracy': 0.7208545804023743,
'specificity': 0.9889052510261536,
'miss_rate': 0.547196090221405,
'fall_out': 0.011094773188233376,
'mcc': 0.5843846797943115},
'test_evaluation': {'loss': 1.027254343032837,
'accuracy': 0.9156197905540466,
'recall': 0.37560975551605225,
'precision': 0.7580872178077698,
'AUROC': 0.8396219611167908,
'AUPRC': 0.5901172757148743,
'f1_score': 0.50232994556427,
'balanced_accuracy': 0.6801413297653198,
'specificity': 0.9846729636192322,
'miss_rate': 0.6243902444839478,
'fall_out': 0.015327036380767822,
'mcc': 0.4960877001285553}},
{'train_evaluation': {'loss': 0.9983038902282715,
'accuracy': 0.9149680137634277,
'recall': 0.2953674793243408,
'precision': 0.867519199848175,
'AUROC': 0.9068061113357544,
'AUPRC': 0.6732707023620605,
'f1_score': 0.44069117307662964,
'balanced_accuracy': 0.6447986364364624,
'specificity': 0.9942298531532288,
'miss_rate': 0.7046325206756592,
'fall_out': 0.005770173389464617,
'mcc': 0.47661104798316956},
'test_evaluation': {'loss': 1.0278276205062866,
'accuracy': 0.9144346714019775,
'recall': 0.28780487179756165,
'precision': 0.8713080286979675,
'AUROC': 0.8591908812522888,
'AUPRC': 0.6105000972747803,
'f1_score': 0.43268728256225586,
'balanced_accuracy': 0.6411845684051514,
'specificity': 0.9945642352104187,
'miss_rate': 0.712195098400116,
'fall_out': 0.005435751285403967,
'mcc': 0.471532940864563}},
{'train_evaluation': {'loss': 1.0349615812301636,
'accuracy': 0.9138816595077515,
'recall': 0.277429461479187,
'precision': 0.8830376863479614,
'AUROC': 0.9181327223777771,
'AUPRC': 0.6828194856643677,
'f1_score': 0.4222104549407959,
'balanced_accuracy': 0.6363643407821655,
'specificity': 0.995299220085144,
'miss_rate': 0.722570538520813,
'fall_out': 0.004700797609984875,
'mcc': 0.4665335416793823},
'test_evaluation': {'loss': 1.0615304708480835,
'accuracy': 0.9139606356620789,
'recall': 0.29128921031951904,
'precision': 0.8530611991882324,
'AUROC': 0.8656487464904785,
'AUPRC': 0.6258153915405273,
'f1_score': 0.4342857003211975,
'balanced_accuracy': 0.6424366235733032,
'specificity': 0.9935840368270874,
'miss_rate': 0.708710789680481,
'fall_out': 0.006415968760848045,
'mcc': 0.46819087862968445}},
{'train_evaluation': {'loss': 0.8717857003211975,
'accuracy': 0.9423836469650269,
'recall': 0.5696621537208557,
'precision': 0.8800107836723328,
'AUROC': 0.9639877080917358,
'AUPRC': 0.8407877087593079,
'f1_score': 0.6916164755821228,
'balanced_accuracy': 0.7798629403114319,
'specificity': 0.9900637269020081,
'miss_rate': 0.4303378760814667,
'fall_out': 0.009936283342540264,
'mcc': 0.6805010437965393},
'test_evaluation': {'loss': 0.983704686164856,
'accuracy': 0.9132496118545532,
'recall': 0.3888501822948456,
'precision': 0.7163029313087463,
'AUROC': 0.8391278982162476,
'AUPRC': 0.5798380374908447,
'f1_score': 0.5040650367736816,
'balanced_accuracy': 0.6845783591270447,
'specificity': 0.9803065657615662,
'miss_rate': 0.6111498475074768,
'fall_out': 0.019693458452820778,
'mcc': 0.48700281977653503}},
{'train_evaluation': {'loss': 1.126129388809204,
'accuracy': 0.9151260256767273,
'recall': 0.29519331455230713,
'precision': 0.8714653253555298,
'AUROC': 0.901314377784729,
'AUPRC': 0.6639528274536133,
'f1_score': 0.44100430607795715,
'balanced_accuracy': 0.6448118686676025,
'specificity': 0.9944303631782532,
'miss_rate': 0.7048066854476929,
'fall_out': 0.005569665227085352,
'mcc': 0.47782906889915466},
'test_evaluation': {'loss': 1.1524351835250854,
'accuracy': 0.9138026237487793,
'recall': 0.2850174307823181,
'precision': 0.8628692030906677,
'AUROC': 0.8497897982597351,
'AUPRC': 0.5952064990997314,
'f1_score': 0.4284965991973877,
'balanced_accuracy': 0.63961261510849,
'specificity': 0.9942077994346619,
'miss_rate': 0.7149825692176819,
'fall_out': 0.0057921940460801125,
'mcc': 0.4662828743457794}},
{'train_evaluation': {'loss': 0.9817284941673279,
'accuracy': 0.925535261631012,
'recall': 0.4334726631641388,
'precision': 0.828010618686676,
'AUROC': 0.9494441151618958,
'AUPRC': 0.769108235836029,
'f1_score': 0.5690443515777588,
'balanced_accuracy': 0.71097731590271,
'specificity': 0.9884819388389587,
'miss_rate': 0.5665273666381836,
'fall_out': 0.011518067680299282,
'mcc': 0.5661799311637878},
'test_evaluation': {'loss': 1.0599387884140015,
'accuracy': 0.9150667786598206,
'recall': 0.3693379759788513,
'precision': 0.7571428418159485,
'AUROC': 0.8406503200531006,
'AUPRC': 0.5868940949440002,
'f1_score': 0.4964871108531952,
'balanced_accuracy': 0.677094578742981,
'specificity': 0.9848511815071106,
'miss_rate': 0.6306620240211487,
'fall_out': 0.015148814767599106,
'mcc': 0.49128806591033936}},
{'train_evaluation': {'loss': 1.5091267824172974,
'accuracy': 0.9035118818283081,
'recall': 0.1588296741247177,
'precision': 0.9431230425834656,
'AUROC': 0.8947315812110901,
'AUPRC': 0.6497679948806763,
'f1_score': 0.2718735933303833,
'balanced_accuracy': 0.5788021683692932,
'specificity': 0.9987746477127075,
'miss_rate': 0.8411703109741211,
'fall_out': 0.001225326326675713,
'mcc': 0.3651188611984253},
'test_evaluation': {'loss': 1.5244096517562866,
'accuracy': 0.9038476943969727,
'recall': 0.16027875244617462,
'precision': 0.9504132270812988,
'AUROC': 0.8565852642059326,
'AUPRC': 0.6051576733589172,
'f1_score': 0.27429935336112976,
'balanced_accuracy': 0.5796047449111938,
'specificity': 0.9989306926727295,
'miss_rate': 0.8397212624549866,
'fall_out': 0.001069328049197793,
'mcc': 0.3685944676399231}},
{'train_evaluation': {'loss': 1.0671989917755127,
'accuracy': 0.9113731384277344,
'recall': 0.24155347049236298,
'precision': 0.9131007194519043,
'AUROC': 0.912755012512207,
'AUPRC': 0.6789199709892273,
'f1_score': 0.3820410370826721,
'balanced_accuracy': 0.6193063259124756,
'specificity': 0.9970592260360718,
'miss_rate': 0.7584465146064758,
'fall_out': 0.002940783277153969,
'mcc': 0.44352811574935913},
'test_evaluation': {'loss': 1.0953367948532104,
'accuracy': 0.9104053378105164,
'recall': 0.23902438580989838,
'precision': 0.8909090757369995,
'AUROC': 0.8570680618286133,
'AUPRC': 0.613454282283783,
'f1_score': 0.3769230842590332,
'balanced_accuracy': 0.6176408529281616,
'specificity': 0.9962573647499084,
'miss_rate': 0.7609755992889404,
'fall_out': 0.0037426482886075974,
'mcc': 0.4343714714050293}},
{'train_evaluation': {'loss': 1.0374341011047363,
'accuracy': 0.9121434688568115,
'recall': 0.25287356972694397,
'precision': 0.9018633365631104,
'AUROC': 0.9072645902633667,
'AUPRC': 0.6683446168899536,
'f1_score': 0.3949945569038391,
'balanced_accuracy': 0.6246767640113831,
'specificity': 0.9964799880981445,
'miss_rate': 0.7471264600753784,
'fall_out': 0.003520028432831168,
'mcc': 0.45062145590782166},
'test_evaluation': {'loss': 1.0641531944274902,
'accuracy': 0.911748468875885,
'recall': 0.2487804889678955,
'precision': 0.9015151262283325,
'AUROC': 0.8561592102050781,
'AUPRC': 0.6101003885269165,
'f1_score': 0.38995084166526794,
'balanced_accuracy': 0.6226525902748108,
'specificity': 0.9965246915817261,
'miss_rate': 0.7512195110321045,
'fall_out': 0.00347531633451581,
'mcc': 0.4467425048351288}},
{'train_evaluation': {'loss': 0.8402834534645081,
'accuracy': 0.9462352991104126,
'recall': 0.6006618142127991,
'precision': 0.8893759846687317,
'AUROC': 0.969927966594696,
'AUPRC': 0.8628208637237549,
'f1_score': 0.7170478105545044,
'balanced_accuracy': 0.795552134513855,
'specificity': 0.9904424548149109,
'miss_rate': 0.3993382155895233,
'fall_out': 0.009557546116411686,
'mcc': 0.704784631729126},
'test_evaluation': {'loss': 0.9914417862892151,
'accuracy': 0.9098522663116455,
'recall': 0.34285715222358704,
'precision': 0.7130434513092041,
'AUROC': 0.8156559467315674,
'AUPRC': 0.5399430394172668,
'f1_score': 0.46305882930755615,
'balanced_accuracy': 0.6626065969467163,
'specificity': 0.982356071472168,
'miss_rate': 0.6571428775787354,
'fall_out': 0.017643913626670837,
'mcc': 0.4541633725166321}}]}
# Saving results
import json
# Saving metrics
file = open("results/MMNN_metrics_enhancers.txt", "w")
string = repr(MMNN_metrics_enhancers)
file.write("MMNN_metrics_enhancers = " + string + "\n")
file.close()
# Saving history
for tuning in ("tuning_True", "tuning_False"):
with open("results/MMNN_history_enhancers_" + tuning + ".json", 'w') as h:
for holdout in range(number_of_splits):
json.dump(MMNN_history_enhancers[tuning][holdout].history, h)
MMNN_distinct_metrics_enhancers = {}
MMNN_distinct_metrics_enhancers["tuning_True"] = {}
MMNN_distinct_metrics_enhancers["tuning_False"] = {}
for tuning in ("tuning_True", "tuning_False"):
MMNN_distinct_metrics_enhancers[tuning]["accuracy_test"] = []
MMNN_distinct_metrics_enhancers[tuning]["AUROC_test"] = []
MMNN_distinct_metrics_enhancers[tuning]["AUPRC_test"] = []
for holdout in MMNN_metrics_enhancers[tuning]:
MMNN_distinct_metrics_enhancers[tuning]["accuracy_test"].append(holdout["test_evaluation"]["accuracy"])
MMNN_distinct_metrics_enhancers[tuning]["AUROC_test"].append(holdout["test_evaluation"]["AUROC"])
MMNN_distinct_metrics_enhancers[tuning]["AUPRC_test"].append(holdout["test_evaluation"]["AUPRC"])
print("MMNN - Enhancers - Wilcoxon test - fixed MLP and CNN performances VS optimized MLP and CNN performances")
print(f"-Accuracy: {wilcoxon(MMNN_distinct_metrics_enhancers['tuning_True']['accuracy_test'], MMNN_distinct_metrics_enhancers['tuning_False']['accuracy_test'])}")
print(f"-AUROC: {wilcoxon(MMNN_distinct_metrics_enhancers['tuning_True']['AUROC_test'], MMNN_distinct_metrics_enhancers['tuning_False']['AUROC_test'])}")
print(f"-AUPRC: {wilcoxon(MMNN_distinct_metrics_enhancers['tuning_True']['AUPRC_test'], MMNN_distinct_metrics_enhancers['tuning_False']['AUPRC_test'])}")
MMNN - Enhancers - Wilcoxon test - fixed MLP and CNN performances VS optimized MLP and CNN performances -Accuracy: WilcoxonResult(statistic=16.0, pvalue=0.275390625) -AUROC: WilcoxonResult(statistic=11.0, pvalue=0.10546875) -AUPRC: WilcoxonResult(statistic=21.0, pvalue=0.556640625)
Hereby all the results from all the different architectures designed are grouped and visualized via barplots.
Note: as stated at the start of this notebook the code was very much revised and customized, hence results were stored in a different format than that necessary for the barplot library used here. Therefore, the following cells are required to format the results for the barplot visualization.
results = {"task": [], "model":[], "accuracy":[], "AUROC":[], "AUPRC":[]}
# MLP
for metrics in (MLP_metrics_promoters, MLP_metrics_enhancers):
for boruta in ("boruta_False", "boruta_True"):
for holdout in (metrics[boruta]):
results["model"].append("MLP_" + str(boruta))
results["accuracy"].append(holdout["accuracy_test"])
results["AUROC"].append(holdout["AUROC_test"])
results["AUPRC"].append(holdout["AUPRC_test"])
if metrics == MLP_metrics_promoters:
results["task"].append("active promoters VS inactive promoters")
else:
results["task"].append("active enhancers VS inactive enhancers")
# CNN
for metrics in (CNN_metrics_promoters, CNN_metrics_enhancers):
for holdout in metrics:
results["model"].append("CNN")
results["accuracy"].append(holdout["test_evaluation"]["accuracy"])
results["AUROC"].append(holdout["test_evaluation"]["AUROC"])
results["AUPRC"].append(holdout["test_evaluation"]["AUPRC"])
if metrics == CNN_metrics_promoters:
results["task"].append("active promoters VS inactive promoters")
else:
results["task"].append("active enhancers VS inactive enhancers")
# MMNN
for metrics in (MMNN_metrics_promoters, MMNN_metrics_enhancers):
for tuning in ("tuning_False", "tuning_True"):
for holdout in metrics[tuning]:
results["model"].append("MMNN_" + str(tuning))
results["accuracy"].append(holdout["test_evaluation"]["accuracy"])
results["AUROC"].append(holdout["test_evaluation"]["AUROC"])
results["AUPRC"].append(holdout["test_evaluation"]["AUPRC"])
if metrics == MMNN_metrics_promoters:
results["task"].append("active promoters VS inactive promoters")
else:
results["task"].append("active enhancers VS inactive enhancers")
resultsDF = pd.DataFrame(results)
resultsDF
| task | model | accuracy | AUROC | AUPRC | |
|---|---|---|---|---|---|
| 0 | active promoters VS inactive promoters | MLP_boruta_False | 0.839315 | 0.896390 | 0.714968 |
| 1 | active promoters VS inactive promoters | MLP_boruta_False | 0.846724 | 0.912573 | 0.736934 |
| 2 | active promoters VS inactive promoters | MLP_boruta_False | 0.848276 | 0.913516 | 0.738965 |
| 3 | active promoters VS inactive promoters | MLP_boruta_False | 0.851379 | 0.916971 | 0.749560 |
| 4 | active promoters VS inactive promoters | MLP_boruta_False | 0.833809 | 0.895597 | 0.705919 |
| ... | ... | ... | ... | ... | ... |
| 95 | active enhancers VS inactive enhancers | MMNN_tuning_True | 0.904559 | 0.843458 | 0.571634 |
| 96 | active enhancers VS inactive enhancers | MMNN_tuning_True | 0.913250 | 0.859116 | 0.602440 |
| 97 | active enhancers VS inactive enhancers | MMNN_tuning_True | 0.904717 | 0.847728 | 0.553511 |
| 98 | active enhancers VS inactive enhancers | MMNN_tuning_True | 0.916173 | 0.858158 | 0.611622 |
| 99 | active enhancers VS inactive enhancers | MMNN_tuning_True | 0.895868 | 0.843695 | 0.500887 |
100 rows × 5 columns
from barplots import barplots
barplots(
resultsDF,
groupby=["task", "model"],
orientation="horizontal",
height=8,
subplots=True
)
((<Figure size 3200x300 with 2 Axes>,
<Figure size 3200x300 with 2 Axes>,
<Figure size 3200x300 with 2 Axes>),
(array([<AxesSubplot:title={'center':'active enhancers VS inactive enhancers'}, xlabel='AUPRC'>,
<AxesSubplot:title={'center':'active promoters VS inactive promoters'}, xlabel='AUPRC'>],
dtype=object),
array([<AxesSubplot:title={'center':'active enhancers VS inactive enhancers'}, xlabel='Accuracy'>,
<AxesSubplot:title={'center':'active promoters VS inactive promoters'}, xlabel='Accuracy'>],
dtype=object),
array([<AxesSubplot:title={'center':'active enhancers VS inactive enhancers'}, xlabel='AUROC'>,
<AxesSubplot:title={'center':'active promoters VS inactive promoters'}, xlabel='AUROC'>],
dtype=object)))
Unit tests for the major functions.
import unittest
class TestNotebook(unittest.TestCase):
def test_knn_imputation(self):
promoters, _ = load_epigenomes(
cell_line = cell_line,
dataset = "fantom",
window_size = window_size,
region = "promoters"
)
enhancers, _ = load_epigenomes(
cell_line = cell_line,
dataset = "fantom",
window_size = window_size,
region = "enhancers"
)
# Testing there are no NaN values after imputation
self.assertEqual(knn_imputation(promoters, 5).isna().values.sum(), 0)
self.assertEqual(knn_imputation(enhancers, 5).isna().values.sum(), 0)
def test_robust_scaler(self):
promoters, _ = load_epigenomes(
cell_line = cell_line,
dataset = "fantom",
window_size = window_size,
region = "promoters"
)
enhancers, _ = load_epigenomes(
cell_line = cell_line,
dataset = "fantom",
window_size = window_size,
region = "enhancers"
)
promoters = robust_scaler(promoters)
enhancers = robust_scaler(enhancers)
feature = "NR2F6"
# Testing the correct application of robust scaling to a feature
self.assertEqual(round(promoters[feature].median()), 0.0)
self.assertEqual(round(promoters[feature].quantile(.75)-promoters[feature].quantile(.25)), 1)
self.assertEqual(round(enhancers[feature].median()), 0.0)
self.assertEqual(round(enhancers[feature].quantile(.75)-enhancers[feature].quantile(.25)), 1)
def test_build_fixed_MLP(self):
MLP = build_fixed_MLP(input_shape=(10, feature_number))
# Testing that layers have the correct number of neurons
self.assertEqual(MLP[1].shape[1], feature_number)
self.assertEqual(MLP[2].shape[1], 64)
def test_build_fixed_CNN(self):
CNN = build_fixed_CNN(window_size=window_size)
# Testing that layers have the correct number of neurons
self.assertEqual(CNN[1].shape[1], 256)
self.assertEqual(CNN[1].shape[2], 4)
self.assertEqual(CNN[2].shape[1], 128)
unittest.main(argv=[''], verbosity=2, exit=False)
test_build_fixed_CNN (__main__.TestNotebook) ...
- Building Fixed CNN:
Model: "model_28"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
sequence_data (InputLayer) [(None, 256, 4)] 0
conv1d_42 (Conv1D) (None, 249, 160) 5280
dropout_112 (Dropout) (None, 249, 160) 0
conv1d_43 (Conv1D) (None, 249, 64) 81984
dropout_113 (Dropout) (None, 249, 64) 0
max_pooling1d_35 (MaxPoolin (None, 124, 64) 0
g1D)
conv1d_44 (Conv1D) (None, 124, 32) 16416
dropout_114 (Dropout) (None, 124, 32) 0
max_pooling1d_36 (MaxPoolin (None, 62, 32) 0
g1D)
conv1d_45 (Conv1D) (None, 62, 16) 3088
dropout_115 (Dropout) (None, 62, 16) 0
max_pooling1d_37 (MaxPoolin (None, 31, 16) 0
g1D)
conv1d_46 (Conv1D) (None, 31, 8) 776
dropout_116 (Dropout) (None, 31, 8) 0
max_pooling1d_38 (MaxPoolin (None, 15, 8) 0
g1D)
conv1d_47 (Conv1D) (None, 15, 8) 392
dropout_117 (Dropout) (None, 15, 8) 0
max_pooling1d_39 (MaxPoolin (None, 7, 8) 0
g1D)
flatten_7 (Flatten) (None, 56) 0
dense_98 (Dense) (None, 128) 7296
dropout_118 (Dropout) (None, 128) 0
dense_99 (Dense) (None, 1) 129
=================================================================
Total params: 115,361
Trainable params: 115,361
Non-trainable params: 0
_________________________________________________________________
ok test_build_fixed_MLP (__main__.TestNotebook) ...
- Building Fixed MLP:
Model: "model_29"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
epigenomic_data (InputLayer [(None, 562)] 0
)
dense_100 (Dense) (None, 128) 72064
dropout_119 (Dropout) (None, 128) 0
dense_101 (Dense) (None, 64) 8256
dropout_120 (Dropout) (None, 64) 0
dense_102 (Dense) (None, 64) 4160
dropout_121 (Dropout) (None, 64) 0
dense_103 (Dense) (None, 1) 65
=================================================================
Total params: 84,545
Trainable params: 84,545
Non-trainable params: 0
_________________________________________________________________
ok test_knn_imputation (__main__.TestNotebook) ... ok test_robust_scaler (__main__.TestNotebook) ... ok ---------------------------------------------------------------------- Ran 4 tests in 50.201s OK
<unittest.main.TestProgram at 0x292d79a9c70>